move internal http endpoints under /internal

docs/administration.md

@@ -232,7 +232,7 @@ Note: This will only work when the replication factor is >= 2
   - List of all indexes on your cluster
   - List of all frames in your indexes
   - Max slice per index, listed in the `/slices/max` endpoint
-- With this information you can query the `/fragment/nodes` endpoint and iterate over each slice
+- With this information you can query the `/internal/fragment/nodes` endpoint and iterate over each slice
 - Using the list of slices owned by this node you will then need to manually:
   - setup a directory structure similar to the other nodes with a path for each Index/Frame
   - copy each owned slice for an existing node to this new node

http/client.go

@@ -81,7 +81,7 @@ func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64
 // maxShardByIndex returns the number of shards on a server by index.
 func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64, error) {
 	// Execute request against the host.
-	u := uriPathToURL(c.defaultURI, "/shards/max")
+	u := uriPathToURL(c.defaultURI, "/internal/shards/max")
 
 	// Build request.
 	req, err := http.NewRequest("GET", u.String(), nil)
@@ -187,7 +187,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo
 // FragmentNodes returns a list of nodes that own a shard.
 func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard uint64) ([]*pilosa.Node, error) {
 	// Execute request against the host.
-	u := uriPathToURL(c.defaultURI, "/fragment/nodes")
+	u := uriPathToURL(c.defaultURI, "/internal/fragment/nodes")
 	u.RawQuery = (url.Values{"index": {index}, "shard": {strconv.FormatUint(shard, 10)}}).Encode()
 
 	// Build request.
@@ -675,7 +675,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pilosa.URI, in
 	if uri == nil {
 		uri = c.defaultURI
 	}
-	u := uriPathToURL(uri, "/fragment/blocks")
+	u := uriPathToURL(uri, "/internal/fragment/blocks")
 	u.RawQuery = url.Values{
 		"index": {index},
 		"field": {field},
@@ -730,7 +730,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pilosa.URI, index,
 		return nil, nil, errors.Wrap(err, "marshaling")
 	}
 
-	u := uriPathToURL(uri, "/fragment/block/data")
+	u := uriPathToURL(uri, "/internal/fragment/block/data")
 	req, err := http.NewRequest("GET", u.String(), bytes.NewReader(buf))
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "creating request")
@@ -770,7 +770,7 @@ func (c *InternalClient) ColumnAttrDiff(ctx context.Context, uri *pilosa.URI, in
 	if uri == nil {
 		uri = c.defaultURI
 	}
-	u := uriPathToURL(uri, fmt.Sprintf("/index/%s/attr/diff", index))
+	u := uriPathToURL(uri, fmt.Sprintf("/internal/index/%s/attr/diff", index))
 
 	// Encode request.
 	buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks})
@@ -814,7 +814,7 @@ func (c *InternalClient) RowAttrDiff(ctx context.Context, uri *pilosa.URI, index
 	if uri == nil {
 		uri = c.defaultURI
 	}
-	u := uriPathToURL(uri, fmt.Sprintf("/index/%s/field/%s/attr/diff", index, field))
+	u := uriPathToURL(uri, fmt.Sprintf("/internal/index/%s/field/%s/attr/diff", index, field))
 
 	// Encode request.
 	buf, err := json.Marshal(postFieldAttrDiffRequest{Blocks: blks})
@@ -862,7 +862,7 @@ func (c *InternalClient) SendMessage(ctx context.Context, uri *pilosa.URI, pb pr
 		return fmt.Errorf("marshaling message: %v", err)
 	}
 
-	u := uriPathToURL(uri, "/cluster/message")
+	u := uriPathToURL(uri, "/internal/cluster/message")
 	req, err := http.NewRequest("POST", u.String(), bytes.NewReader(msg))
 	if err != nil {
 		return errors.Wrap(err, "making new request")

http/handler.go

@@ -189,39 +189,38 @@ func (h *Handler) queryArgValidator(next http.Handler) http.Handler {
 func NewRouter(handler *Handler) *mux.Router {
 	router := mux.NewRouter()
 	router.HandleFunc("/", handler.handleHome).Methods("GET")
-	router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
-	router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
-
-	router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
-	router.HandleFunc("/cluster/message", handler.handlePostClusterMessage).Methods("POST")
-	router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST")
-	router.HandleFunc("/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client
-	router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
-	router.HandleFunc("/info", handler.handleGetInfo).Methods("GET")
-	router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
-
 	router.HandleFunc("/cluster/resize/abort", handler.handlePostClusterResizeAbort).Methods("POST")
-
 	router.HandleFunc("/cluster/resize/remove-node", handler.handlePostClusterResizeRemoveNode).Methods("POST")
+	router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST")
 	router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
 	router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
 	router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport")
-	router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET")
-	router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
-	router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
 	router.HandleFunc("/import", handler.handlePostImport).Methods("POST")
 	router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST")
 	router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET")
 	router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET")
 	router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST")
 	router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE")
-	router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST")
 	//router.HandleFunc("/index/{index}/field", handler.handleGetFields).Methods("GET") // Not implemented.
 	router.HandleFunc("/index/{index}/field/{field}", handler.handlePostField).Methods("POST")
 	router.HandleFunc("/index/{index}/field/{field}", handler.handleDeleteField).Methods("DELETE")
-	router.HandleFunc("/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST")
 	router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST").Name("PostQuery")
+	router.HandleFunc("/info", handler.handleGetInfo).Methods("GET")
 	router.HandleFunc("/recalculate-caches", handler.handleRecalculateCaches).Methods("POST")
+	router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
+	router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
+	router.HandleFunc("/version", handler.handleGetVersion).Methods("GET")
+
+	// /internal endpoints are for internal use only; they may change at any time.
+	// DO NOT rely on these for external applications!
+	router.HandleFunc("/internal/cluster/message", handler.handlePostClusterMessage).Methods("POST")
+	router.HandleFunc("/internal/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET")
+	router.HandleFunc("/internal/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
+	router.HandleFunc("/internal/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
+	router.HandleFunc("/internal/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST")
+	router.HandleFunc("/internal/index/{index}/field/{field}/attr/diff", handler.handlePostFieldAttrDiff).Methods("POST")
+	router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET") // TODO: deprecate, but it's being used by the client
+	router.HandleFunc("/internal/translate/data", handler.handleGetTranslateData).Methods("GET")
 
 	// TODO: Apply MethodNotAllowed statuses to all endpoints.
 	// Ideally this would be automatic, as described in this (wontfix) ticket:
@@ -229,8 +228,6 @@ func NewRouter(handler *Handler) *mux.Router {
 	// For now we just do it for the most commonly used handler, /query
 	router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET")
 
-	router.HandleFunc("/translate/data", handler.handleGetTranslateData).Methods("GET")
-
 	router.Use(handler.queryArgValidator)
 	return router
 }
@@ -441,7 +438,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// handleGetShardsMax handles GET /shards/max requests.
+// handleGetShardsMax handles GET /internal/shards/max requests.
 func (h *Handler) handleGetShardsMax(w http.ResponseWriter, r *http.Request) {
 	if !validHeaderAcceptJSON(r.Header) {
 		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
@@ -590,7 +587,7 @@ func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
 	resp.write(w, err)
 }
 
-// handlePostIndexAttrDiff handles POST /index/attr/diff requests.
+// handlePostIndexAttrDiff handles POST /internal/index/attr/diff requests.
 func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) {
 	if !validHeaderAcceptJSON(r.Header) {
 		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
@@ -760,7 +757,7 @@ func (h *Handler) handleDeleteField(w http.ResponseWriter, r *http.Request) {
 	resp.write(w, err)
 }
 
-// handlePostFieldAttrDiff handles POST /field/attr/diff requests.
+// handlePostFieldAttrDiff handles POST /internal/field/attr/diff requests.
 func (h *Handler) handlePostFieldAttrDiff(w http.ResponseWriter, r *http.Request) {
 	if !validHeaderAcceptJSON(r.Header) {
 		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
@@ -1019,7 +1016,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// handleGetFragmentNodes handles /fragment/nodes requests.
+// handleGetFragmentNodes handles /internal/fragment/nodes requests.
 func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) {
 	if !validHeaderAcceptJSON(r.Header) {
 		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
@@ -1048,7 +1045,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// handleGetFragmentBlockData handles GET /fragment/block/data requests.
+// handleGetFragmentBlockData handles GET /internal/fragment/block/data requests.
 func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) {
 	buf, err := h.API.FragmentBlockData(r.Context(), r.Body)
 	if err != nil {
@@ -1068,7 +1065,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ
 	w.Write(buf)
 }
 
-// handleGetFragmentBlocks handles GET /fragment/blocks requests.
+// handleGetFragmentBlocks handles GET /internal/fragment/blocks requests.
 func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request) {
 	if !validHeaderAcceptJSON(r.Header) {
 		http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)

http/translator.go

@@ -54,7 +54,7 @@ func (s *TranslateStore) Reader(ctx context.Context, off int64) (io.ReadCloser,
 	if err != nil {
 		return nil, err
 	}
-	u.Path = "/translate/data"
+	u.Path = "/internal/translate/data"
 	u.RawQuery = (url.Values{
 		"offset": {strconv.FormatInt(off, 10)},
 	}).Encode()

server/handler_test.go

@@ -124,7 +124,7 @@ func TestHandler_Endpoints(t *testing.T) {
 
 	t.Run("Max Shard", func(t *testing.T) {
 		w := httptest.NewRecorder()
-		h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/shards/max", nil))
+		h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/internal/shards/max", nil))
 		if w.Code != gohttp.StatusOK {
 			t.Fatalf("unexpected status code: %d", w.Code)
 		} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" {
@@ -436,7 +436,7 @@ func TestHandler_Endpoints(t *testing.T) {
 		// Send block checksums to determine diff.
 		req := test.MustNewHTTPRequest(
 			"POST",
-			"/index/i/attr/diff",
+			"/internal/index/i/attr/diff",
 			strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
 		)
 		req.Header.Set("Content-Type", "application/json")
@@ -476,7 +476,7 @@ func TestHandler_Endpoints(t *testing.T) {
 		// Send block checksums to determine diff.
 		req := test.MustNewHTTPRequest(
 			"POST",
-			"/index/i/field/meta/attr/diff",
+			"/internal/index/i/field/meta/attr/diff",
 			strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
 		)
 		req.Header.Set("Content-Type", "application/json")
@@ -507,7 +507,7 @@ func TestHandler_Endpoints(t *testing.T) {
 
 	t.Run("Fragment Nodes", func(t *testing.T) {
 		w := httptest.NewRecorder()
-		r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=i&shard=0", nil)
+		r := test.MustNewHTTPRequest("GET", "/internal/fragment/nodes?index=i&shard=0", nil)
 		h.ServeHTTP(w, r)
 		if w.Code != gohttp.StatusOK {
 			t.Fatalf("unexpected status code: %d", w.Code)
@@ -520,7 +520,7 @@ func TestHandler_Endpoints(t *testing.T) {
 
 		// invalid argument should return BadRequest
 		w = httptest.NewRecorder()
-		r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&shard=0", nil)
+		r = test.MustNewHTTPRequest("GET", "/internal/fragment/nodes?db=X&shard=0", nil)
 		h.ServeHTTP(w, r)
 		if w.Code != gohttp.StatusBadRequest {
 			t.Fatalf("unexpected status code: %d", w.Code)
@@ -528,7 +528,7 @@ func TestHandler_Endpoints(t *testing.T) {
 
 		// index is required
 		w = httptest.NewRecorder()
-		r = test.MustNewHTTPRequest("GET", "/fragment/nodes?shard=0", nil)
+		r = test.MustNewHTTPRequest("GET", "/internal/fragment/nodes?shard=0", nil)
 		h.ServeHTTP(w, r)
 		if w.Code != gohttp.StatusBadRequest {
 			t.Fatalf("unexpected status code: %d", w.Code)