From cba91be126b3a8bc391b6aafde97144fd1a9cd92 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Sat, 30 Jun 2018 17:43:49 -0500 Subject: [PATCH] move internal http endpoints under /internal --- docs/administration.md | 2 +- http/client.go | 14 ++++++------- http/handler.go | 47 ++++++++++++++++++++---------------------- http/translator.go | 2 +- server/handler_test.go | 12 +++++------ 5 files changed, 37 insertions(+), 40 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index dc2e923f2..accbc46de 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -232,7 +232,7 @@ Note: This will only work when the replication factor is >= 2 - List of all indexes on your cluster - List of all frames in your indexes - Max slice per index, listed in the `/slices/max` endpoint -- With this information you can query the `/fragment/nodes` endpoint and iterate over each slice +- With this information you can query the `/internal/fragment/nodes` endpoint and iterate over each slice - Using the list of slices owned by this node you will then need to manually: - setup a directory structure similar to the other nodes with a path for each Index/Frame - copy each owned slice for an existing node to this new node diff --git a/http/client.go b/http/client.go index 67b9135a5..c56e50721 100644 --- a/http/client.go +++ b/http/client.go @@ -81,7 +81,7 @@ func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64 // maxShardByIndex returns the number of shards on a server by index. func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64, error) { // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/shards/max") + u := uriPathToURL(c.defaultURI, "/internal/shards/max") // Build request. req, err := http.NewRequest("GET", u.String(), nil) @@ -187,7 +187,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo // FragmentNodes returns a list of nodes that own a shard. func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*pilosa.Node, error) { // Execute request against the host. - u := uriPathToURL(c.defaultURI, "/fragment/nodes") + u := uriPathToURL(c.defaultURI, "/internal/fragment/nodes") u.RawQuery = (url.Values{"index": {index}, "shard": {strconv.FormatUint(shard, 10)}}).Encode() // Build request. @@ -675,7 +675,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in if uri == nil { uri = c.defaultURI } - u := uriPathToURL(uri, "/fragment/blocks") + u := uriPathToURL(uri, "/internal/fragment/blocks") u.RawQuery = url.Values{ "index": {index}, "field": {field}, @@ -730,7 +730,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index, return nil, nil, errors.Wrap(err, "marshaling") } - u := uriPathToURL(uri, "/fragment/block/data") + u := uriPathToURL(uri, "/internal/fragment/block/data") req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf)) if err != nil { return nil, nil, errors.Wrap(err, "creating request") @@ -770,7 +770,7 @@ func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, in if uri == nil { uri = c.defaultURI } - u := uriPathToURL(uri, fmt.Sprintf("/index/%s/attr/diff", index)) + u := uriPathToURL(uri, fmt.Sprintf("/internal/index/%s/attr/diff", index)) // Encode request. buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks}) @@ -814,7 +814,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index if uri == nil { uri = c.defaultURI } - u := uriPathToURL(uri, fmt.Sprintf("/index/%s/field/%s/attr/diff", index, field)) + u := uriPathToURL(uri, fmt.Sprintf("/internal/index/%s/field/%s/attr/diff", index, field)) // Encode request. buf, err := json.Marshal(postFieldAttrDiffRequest{Blocks: blks}) @@ -862,7 +862,7 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb pr return fmt.Errorf("marshaling message: %v", err) } - u := uriPathToURL(uri, "/cluster/message") + u := uriPathToURL(uri, "/internal/cluster/message") req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg)) if err != nil { return errors.Wrap(err, "making new request") diff --git a/http/handler.go b/http/handler.go index d8067c378..a24beb34a 100644 --- a/http/handler.go +++ b/http/handler.go @@ -189,39 +189,38 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler { func NewRouter(handler *Handler) *mux.Router { router := mux.NewRouter() router.HandleFunc("/", handler.handleHome).Methods("GET") - router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") - router.Handle("/debug/vars", expvar.Handler()).Methods("GET") - - router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") - router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST") - router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") - router.HandleFunc("/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") - router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") - router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") - router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST") - router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST") + router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.Handle("/debug/vars", expvar.Handler()).Methods("GET") router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") - router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") - router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") - router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") router.HandleFunc("/import", handler.handlePostImport).Methods("POST") router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST") router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET") router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET") router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST") router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE") - router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") //router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented. router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST") router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE") - router.HandleFunc("/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST") router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery") + router.HandleFunc("/info", handler.handleGetInfo).Methods("GET") router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST") + router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") + router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") + router.HandleFunc("/version", handler.handleGetVersion).Methods("GET") + + // /internal endpoints are for internal use only; they may change at any time. + // DO NOT rely on these for external applications! + router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST") + router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") + router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") + router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") + router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST") + router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST") + router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client + router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET") // TODO: Apply MethodNotAllowed statuses to all endpoints. // Ideally this would be automatic, as described in this (wontfix) ticket: @@ -229,8 +228,6 @@ func NewRouter(handler *Handler) *mux.Router { // For now we just do it for the most commonly used handler, /query router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET") - router.HandleFunc("/translate/data", handler.handleGetTranslateData).Methods("GET") - router.Use(handler.queryArgValidator) return router } @@ -441,7 +438,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } -// handleGetShardsMax handles GET /shards/max requests. +// handleGetShardsMax handles GET /internal/shards/max requests. func (h *Handler) handleGetShardsMax(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) @@ -590,7 +587,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) { resp.write(w, err) } -// handlePostIndexAttrDiff handles POST /index/attr/diff requests. +// handlePostIndexAttrDiff handles POST /internal/index/attr/diff requests. func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) @@ -760,7 +757,7 @@ func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) { resp.write(w, err) } -// handlePostFieldAttrDiff handles POST /field/attr/diff requests. +// handlePostFieldAttrDiff handles POST /internal/field/attr/diff requests. func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) @@ -1019,7 +1016,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { } } -// handleGetFragmentNodes handles /fragment/nodes requests. +// handleGetFragmentNodes handles /internal/fragment/nodes requests. func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) @@ -1048,7 +1045,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } } -// handleGetFragmentBlockData handles GET /fragment/block/data requests. +// handleGetFragmentBlockData handles GET /internal/fragment/block/data requests. func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { buf, err := h.API.FragmentBlockData(r.Context(), r.Body) if err != nil { @@ -1068,7 +1065,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ w.Write(buf) } -// handleGetFragmentBlocks handles GET /fragment/blocks requests. +// handleGetFragmentBlocks handles GET /internal/fragment/blocks requests. func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) diff --git a/http/translator.go b/http/translator.go index 3ca9b840d..08235bf47 100644 --- a/http/translator.go +++ b/http/translator.go @@ -54,7 +54,7 @@ func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser, if err != nil { return nil, err } - u.Path = "/translate/data" + u.Path = "/internal/translate/data" u.RawQuery = (url.Values{ "offset": {strconv.FormatInt(off, 10)}, }).Encode() diff --git a/server/handler_test.go b/server/handler_test.go index 00326059e..e49b1eb0f 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -124,7 +124,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Max Shard", func(t *testing.T) { w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/shards/max", nil)) + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/internal/shards/max", nil)) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { @@ -436,7 +436,7 @@ func TestHandler_Endpoints(t *testing.T) { // Send block checksums to determine diff. req := test.MustNewHTTPRequest( "POST", - "/index/i/attr/diff", + "/internal/index/i/attr/diff", strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), ) req.Header.Set("Content-Type", "application/json") @@ -476,7 +476,7 @@ func TestHandler_Endpoints(t *testing.T) { // Send block checksums to determine diff. req := test.MustNewHTTPRequest( "POST", - "/index/i/field/meta/attr/diff", + "/internal/index/i/field/meta/attr/diff", strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`), ) req.Header.Set("Content-Type", "application/json") @@ -507,7 +507,7 @@ func TestHandler_Endpoints(t *testing.T) { t.Run("Fragment Nodes", func(t *testing.T) { w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=i&shard=0", nil) + r := test.MustNewHTTPRequest("GET", "/internal/fragment/nodes?index=i&shard=0", nil) h.ServeHTTP(w, r) if w.Code != gohttp.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) @@ -520,7 +520,7 @@ func TestHandler_Endpoints(t *testing.T) { // invalid argument should return BadRequest w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&shard=0", nil) + r = test.MustNewHTTPRequest("GET", "/internal/fragment/nodes?db=X&shard=0", nil) h.ServeHTTP(w, r) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code) @@ -528,7 +528,7 @@ func TestHandler_Endpoints(t *testing.T) { // index is required w = httptest.NewRecorder() - r = test.MustNewHTTPRequest("GET", "/fragment/nodes?shard=0", nil) + r = test.MustNewHTTPRequest("GET", "/internal/fragment/nodes?shard=0", nil) h.ServeHTTP(w, r) if w.Code != gohttp.StatusBadRequest { t.Fatalf("unexpected status code: %d", w.Code)