From c36dd395e664be375d37fe70f9988d5104a20d57 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 7 Mar 2018 15:39:44 +0300 Subject: [PATCH] All tests pass --- api.go | 7 +- client_test.go | 67 ++++++++++--------- ctl/backup_test.go | 10 ++- ctl/export_test.go | 10 ++- ctl/import_test.go | 21 +++--- ctl/restore_test.go | 10 ++- executor_test.go | 10 +-- handler.go | 67 +++++++++++-------- handler_test.go | 158 ++++++++++++++++++++++---------------------- holder_test.go | 2 +- stats_test.go | 16 ++--- test/handler.go | 33 ++++++--- 12 files changed, 218 insertions(+), 193 deletions(-) diff --git a/api.go b/api.go index f369ba10a..97d7828e5 100644 --- a/api.go +++ b/api.go @@ -61,7 +61,6 @@ func (a *API) ExecuteQuery(ctx context.Context, req *QueryRequest) (QueryRespons q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() if err != nil { - // TODO: Wrap return resp, err } execOpts := &ExecOptions{ @@ -735,10 +734,16 @@ func (api *API) SliceMax(ctx context.Context, inverse bool) map[string]uint64 { } func (api *API) StatsWithTags(tags []string) StatsClient { + if api.Holder == nil || api.Cluster == nil { + return nil + } return api.Holder.Stats.WithTags(tags...) } func (api *API) ClusterLongQueryTime() time.Duration { + if api.Cluster == nil { + return 0 + } return api.Cluster.LongQueryTime } diff --git a/client_test.go b/client_test.go index f2bd18e50..36b9dc38d 100644 --- a/client_test.go +++ b/client_test.go @@ -36,10 +36,10 @@ func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) { for i := 0; i < numNodes; i++ { hldr[i] = test.MustOpenHolder() server[i] = test.NewServer() - server[i].Handler.Cluster = c - server[i].Handler.Cluster.Nodes[i].URI = server[i].HostURI() - server[i].Handler.Holder = hldr[i].Holder - server[i].Handler.Node = server[i].Handler.Cluster.Nodes[i] + server[i].Handler.API.URI = server[i].HostURI() + server[i].Handler.API.Cluster = c + server[i].Handler.API.Cluster.Nodes[i].URI = server[i].HostURI() + server[i].Handler.API.Holder = hldr[i].Holder } return server, hldr } @@ -86,7 +86,7 @@ func TestClient_MultiNode(t *testing.T) { // Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN. sliceNums := []uint64{1, 2, 6} for i, num := range sliceNums { - owns := s[i].Handler.Handler.Cluster.OwnsSlices("i", 20, s[i].HostURI()) + owns := s[i].Handler.Handler.API.Cluster.OwnsSlices("i", 20, s[i].Host()) ownsNum := false for _, ownNum := range owns { if ownNum == num { @@ -217,10 +217,10 @@ func TestClient_Import(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -268,10 +268,10 @@ func TestClient_ImportInverseEnabled(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -317,10 +317,10 @@ func TestClient_ImportValue(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Send import request. c := test.MustNewClient(s.Host(), defaultClient) @@ -355,10 +355,10 @@ func TestClient_BackupRestore(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder c := test.MustNewClient(s.Host(), defaultClient) @@ -420,10 +420,11 @@ func TestClient_BackupInverseView(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder c := test.MustNewClient(s.Host(), defaultClient) @@ -457,10 +458,10 @@ func TestClient_BackupInvalidView(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder c := test.MustNewClient(s.Host(), defaultClient) @@ -486,10 +487,10 @@ func TestClient_FragmentBlocks(t *testing.T) { s := test.NewServer() defer s.Close() - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.Holder = hldr.Holder - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.URI = s.HostURI() + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder // Retrieve blocks. c := test.MustNewClient(s.Host(), defaultClient) diff --git a/ctl/backup_test.go b/ctl/backup_test.go index d23b1a917..7a4feb3d9 100644 --- a/ctl/backup_test.go +++ b/ctl/backup_test.go @@ -50,12 +50,10 @@ func TestBackupCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = *uri - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm := NewBackupCommand(stdin, stdout, stderr) file, err := ioutil.TempFile("", "import.csv") diff --git a/ctl/export_test.go b/ctl/export_test.go index 5d1d5a4d7..2bcccd6ba 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -63,12 +63,10 @@ func TestExportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0] = node - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm.Host = s.Host() http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) diff --git a/ctl/import_test.go b/ctl/import_test.go index 9022efae9..119a56071 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -69,12 +69,10 @@ func TestImportCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0] = node - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm.Host = s.Host() cm.Index = "i" @@ -111,16 +109,15 @@ func TestImportCommand_RunValue(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0] = node - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm.Host = s.Host() http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader(""))) - http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(""))) + http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader(`{"options":{"rangeEnabled": true, "fields": [{"name": "foo", "type": "int", "min": 0, "max": 100}]}}`))) cm.Index = "i" cm.Frame = "f" diff --git a/ctl/restore_test.go b/ctl/restore_test.go index bb8eb3b5f..f1edb7fe9 100644 --- a/ctl/restore_test.go +++ b/ctl/restore_test.go @@ -52,12 +52,10 @@ func TestRestoreCommand_Run(t *testing.T) { if err != nil { t.Fatal(err) } - node := &pilosa.Node{ID: "node", URI: *uri} - - s.Handler.Node = node - s.Handler.Cluster = test.NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = *uri - s.Handler.Holder = hldr.Holder + s.Handler.API.URI = uri + s.Handler.API.Cluster = test.NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() + s.Handler.API.Holder = hldr.Holder cm := NewRestoreCommand(stdin, stdout, stderr) cm.Path = file.Name() diff --git a/executor_test.go b/executor_test.go index c98239137..690bfc5ce 100644 --- a/executor_test.go +++ b/executor_test.go @@ -927,7 +927,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { // The local node owns slice 1. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := test.NewExecutor(hldr.Holder, c) @@ -961,7 +961,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { // Create local executor data. The local node owns slice 1. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) @@ -1004,7 +1004,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Create local executor data. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder // Create frame. if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { @@ -1056,7 +1056,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Create local executor data. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder // Create frame. if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { @@ -1130,7 +1130,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { // Create local executor data on slice 2 & 4. hldr := test.MustOpenHolder() defer hldr.Close() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) diff --git a/handler.go b/handler.go index 0f04028e0..f6a385ccb 100644 --- a/handler.go +++ b/handler.go @@ -257,9 +257,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // useragent tag identifies internal/external endpoints statsTags = append(statsTags, "useragent:"+r.UserAgent()) - stats := h.API.StatsWithTags(statsTags) - stats.Histogram("http."+endpointName, float64(dif), 0.1) + if stats != nil { + stats.Histogram("http."+endpointName, float64(dif), 0.1) + } } func (h *Handler) handleWebUI(w http.ResponseWriter, r *http.Request) { @@ -328,7 +329,8 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { resp, err := h.API.ExecuteQuery(r.Context(), req) if err != nil { w.WriteHeader(http.StatusBadRequest) - h.writeQueryResponse(w, r, &resp) + h.writeQueryResponse(w, r, &QueryResponse{Err: err}) + return } // Set appropriate status code, if there is an error. @@ -348,10 +350,15 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) { - inverse, err := strconv.ParseBool(r.URL.Query().Get("inverse")) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return + var err error + inverse := false + inverseStr := r.URL.Query().Get("inverse") + if inverseStr != "" { + inverse, err = strconv.ParseBool(r.URL.Query().Get("inverse")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } } ms := h.API.SliceMax(r.Context(), inverse) if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { @@ -1161,7 +1168,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { if err = h.API.ExportCSV(r.Context(), index, frame, view, slice, w); err != nil { switch err { case ErrFragmentNotFound: - http.Error(w, err.Error(), http.StatusNotFound) + break case ErrClusterDoesNotOwnSlice: http.Error(w, err.Error(), http.StatusPreconditionFailed) default: @@ -1528,26 +1535,30 @@ func (h *Handler) handlePostInputDefinition(w http.ResponseWriter, r *http.Reque return } - err = h.API.CreateInputDefinition(r.Context(), indexName, inputDefName, req) - switch err { - case nil: - break - case ErrIndexNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case ErrInputDefinitionExists: - http.Error(w, err.Error(), http.StatusConflict) - case ErrInputDefinitionAttrsRequired: - fallthrough - case ErrInputDefinitionNameRequired: - fallthrough - case ErrInputDefinitionActionRequired: - fallthrough - case ErrInputDefinitionHasPrimaryKey: - fallthrough - case ErrInputDefinitionDupePrimaryKey: - http.Error(w, err.Error(), http.StatusBadRequest) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) + if err = h.API.CreateInputDefinition(r.Context(), indexName, inputDefName, req); err != nil { + switch err { + case ErrIndexNotFound: + http.Error(w, err.Error(), http.StatusNotFound) + case ErrInputDefinitionExists: + http.Error(w, err.Error(), http.StatusConflict) + case ErrInputDefinitionAttrsRequired: + fallthrough + case ErrInputDefinitionNameRequired: + fallthrough + case ErrInputDefinitionActionRequired: + fallthrough + case ErrInputDefinitionHasPrimaryKey: + fallthrough + case ErrInputDefinitionDupePrimaryKey: + http.Error(w, err.Error(), http.StatusBadRequest) + default: + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + if err := json.NewEncoder(w).Encode(defaultInputDefinitionResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) } } diff --git a/handler_test.go b/handler_test.go index 1e30f41ad..39bcc7fff 100644 --- a/handler_test.go +++ b/handler_test.go @@ -65,8 +65,8 @@ func TestHandler_NotFound(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil)) @@ -100,8 +100,8 @@ func TestHandler_Schema(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { @@ -139,9 +139,9 @@ func TestHandler_Status(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) - h.StatusHandler = s + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.StatusHandler = s s.Handler = h w := httptest.NewRecorder() @@ -186,8 +186,8 @@ func TestHandler_MaxSlices(t *testing.T) { hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { @@ -227,8 +227,8 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) if w.Code != http.StatusOK { @@ -244,8 +244,8 @@ func TestHandler_Query_Args_URL(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "idx0" { t.Fatalf("unexpected index: %s", index) @@ -272,8 +272,8 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if index != "idx0" { t.Fatalf("unexpected index: %s", index) @@ -312,8 +312,8 @@ func TestHandler_Query_Args_Err(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusBadRequest { @@ -339,8 +339,8 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } @@ -360,8 +360,8 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{uint64(100)}, nil } @@ -390,8 +390,8 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -423,8 +423,8 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} @@ -446,8 +446,8 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -494,8 +494,8 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} @@ -555,8 +555,8 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, @@ -579,8 +579,8 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return []interface{}{[]pilosa.Pair{ {ID: 1, Count: 2}, @@ -612,15 +612,15 @@ func TestHandler_Query_Err_JSON(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`))) - if w.Code != http.StatusInternalServerError { + if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" { t.Fatalf("unexpected body: %q", body) @@ -633,8 +633,8 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { return nil, errors.New("marker") } @@ -643,7 +643,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)) r.Header.Set("Accept", "application/x-protobuf") h.ServeHTTP(w, r) - if w.Code != http.StatusInternalServerError { + if w.Code != http.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) } @@ -661,8 +661,8 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil)) if w.Code != http.StatusMethodNotAllowed { @@ -676,8 +676,8 @@ func TestHandler_Query_ErrParse(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn("))) if w.Code != http.StatusBadRequest { @@ -693,7 +693,7 @@ func TestHandler_Index_Delete(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Create index. @@ -733,8 +733,8 @@ func TestHandler_DeleteFrame(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader(""))) if w.Code != http.StatusOK { @@ -753,8 +753,8 @@ func TestHandler_SetIndexTimeQuantum(t *testing.T) { hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { @@ -777,8 +777,8 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) { } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`))) if w.Code != http.StatusOK { @@ -796,7 +796,7 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Set attributes on the index. @@ -845,7 +845,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Set attributes on the index. @@ -895,7 +895,7 @@ func TestHandler_Frame_AddField(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() t.Run("OK", func(t *testing.T) { @@ -999,7 +999,7 @@ func TestHandler_Frame_DeleteField(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() t.Run("OK", func(t *testing.T) { @@ -1064,7 +1064,7 @@ func TestHandler_Frame_GetFields(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() t.Run("OK", func(t *testing.T) { @@ -1132,7 +1132,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Set bits in the index. @@ -1181,8 +1181,8 @@ func TestHandler_Version(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/version", nil) @@ -1204,9 +1204,9 @@ func TestHandler_Fragment_Nodes(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(3) - h.Cluster.ReplicaN = 2 + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(3) + h.API.Cluster.ReplicaN = 2 w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil) @@ -1233,8 +1233,8 @@ func TestHandler_Expvars(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Cluster = test.NewCluster(1) - h.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder w := httptest.NewRecorder() r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) h.ServeHTTP(w, r) @@ -1279,8 +1279,8 @@ func TestHandler_CreateInputDefinition(t *testing.T) { ] }`) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input-definition/input1", bytes.NewBuffer(inputBody))) if w.Code != http.StatusOK { @@ -1314,8 +1314,8 @@ func TestHandler_DuplicatePrimaryKey(t *testing.T) { defer hldr.Close() hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) //Ensure throwing error if there's duplicated primaryKey field invalidPrimaryKey := []byte(` @@ -1419,8 +1419,8 @@ func TestHandler_DeleteInputDefinition(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) // Test index not found. w := httptest.NewRecorder() @@ -1467,8 +1467,8 @@ func TestHandler_GetInputDefinition(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) frames := internal.Frame{Name: "f", Meta: &internal.FrameMeta{RowLabel: "row"}} action := internal.InputDefinitionAction{Frame: "f", ValueDestination: "mapping", ValueMap: map[string]uint64{"Green": 1}} @@ -1634,8 +1634,8 @@ func TestHandler_CreateInput(t *testing.T) { "null_value": null }]`) h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) // Return error if index does not exist. w := httptest.NewRecorder() @@ -1749,8 +1749,8 @@ func TestInput_JSON(t *testing.T) { err: "set-timestamp value must be in time format: YYYY-MM-DD, has: 12345"}, } h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) for _, req := range tests { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/input/input1", bytes.NewBuffer([]byte(req.json)))) @@ -1810,8 +1810,8 @@ func TestHandler_DeleteView(t *testing.T) { hldr.Index("i0").Frame("f0").SetTimeQuantum("YMD") h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f0/view/standard_2017", strings.NewReader(""))) if w.Code != http.StatusOK { @@ -1836,8 +1836,8 @@ func TestHandler_RecalculateCaches(t *testing.T) { defer hldr.Close() h := test.NewHandler() - h.Holder = hldr.Holder - h.Cluster = test.NewCluster(1) + h.API.Holder = hldr.Holder + h.API.Cluster = test.NewCluster(1) w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) diff --git a/holder_test.go b/holder_test.go index 41ede1adf..4c63e6e76 100644 --- a/holder_test.go +++ b/holder_test.go @@ -393,7 +393,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { defer hldr1.Close() s := test.NewServer() defer s.Close() - s.Handler.Holder = hldr1.Holder + s.Handler.API.Holder = hldr1.Holder s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { e := pilosa.NewExecutor(client) e.Holder = hldr1.Holder diff --git a/stats_test.go b/stats_test.go index 60deae993..591236268 100644 --- a/stats_test.go +++ b/stats_test.go @@ -215,10 +215,10 @@ func TestStatsCount_CreateIndex(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() called := false - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { if name != "createIndex" { t.Errorf("Expected createIndex, Results %s", name) @@ -239,7 +239,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Create index. @@ -247,7 +247,7 @@ func TestStatsCount_DeleteIndex(t *testing.T) { t.Fatal(err) } called := false - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { if name != "deleteIndex" { t.Errorf("Expected deleteIndex, Results %s", name) @@ -268,7 +268,7 @@ func TestStatsCount_CreateFrame(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() // Create index. @@ -276,7 +276,7 @@ func TestStatsCount_CreateFrame(t *testing.T) { t.Fatal(err) } called := false - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, index []string) { if name != "createFrame" { t.Errorf("Expected createFrame, Results %s", name) @@ -300,7 +300,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { defer hldr.Close() s := test.NewServer() - s.Handler.Holder = hldr.Holder + s.Handler.API.Holder = hldr.Holder defer s.Close() called := false // Create index. @@ -308,7 +308,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) { if _, err := indx.CreateFrameIfNotExists("test", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - s.Handler.Holder.Stats = &MockStats{ + s.Handler.API.Holder.Stats = &MockStats{ mockCountWithTags: func(name string, value int64, rate float64, index []string) { if name != "deleteFrame" { t.Errorf("Expected deleteFrame, Results %s", name) diff --git a/test/handler.go b/test/handler.go index 0f8c1cefd..ae7605603 100644 --- a/test/handler.go +++ b/test/handler.go @@ -27,6 +27,7 @@ import ( "github.com/pilosa/pilosa" "github.com/pilosa/pilosa/internal" "github.com/pilosa/pilosa/pql" + "github.com/pkg/errors" ) // Handler represents a test wrapper for pilosa.Handler. @@ -40,10 +41,12 @@ func NewHandler() *Handler { h := &Handler{ Handler: pilosa.NewHandler(), } - h.Handler.Executor = &h.Executor + h.API = pilosa.NewAPI(nil) + h.Handler.API = pilosa.NewAPI(nil) + h.Handler.API.Executor = &h.Executor // Handler test messages can no-op. - h.Broadcaster = pilosa.NopBroadcaster + h.API.Broadcaster = pilosa.NopBroadcaster h.SetNormal() @@ -80,21 +83,35 @@ func NewServer() *Server { if err != nil { panic(err) } + s.Handler.API.URI = uri // Handler test messages can no-op. - s.Handler.Broadcaster = pilosa.NopBroadcaster + s.Handler.API.Broadcaster = pilosa.NopBroadcaster // Create a default cluster on the handler - s.Handler.Cluster = NewCluster(1) - s.Handler.Cluster.Nodes[0].URI = *uri - - s.Handler.Node = s.Handler.Cluster.Nodes[0] + s.Handler.API.Cluster = NewCluster(1) + s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() return s } // LocalStatus exists so that test.Server implements StatusHandler. func (s *Server) LocalStatus() (proto.Message, error) { - return nil, nil + if s.Handler.API.Holder == nil { + return nil, errors.New("Server.Holder is nil") + } + + ns := internal.NodeStatus{ + Host: s.Handler.Handler.API.URI.HostPort(), + State: pilosa.NodeStateUp, + Indexes: pilosa.EncodeIndexes(s.Handler.API.Holder.Indexes()), + } + + // Append Slice list per this Node's indexes + for _, index := range ns.Indexes { + index.Slices = s.Handler.API.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.API.URI.HostPort()) + } + + return &ns, nil } // ClusterStatus exists so that test.Server implements StatusHandler.