From 664c911791cd1532097fac6140ad5b1172776432 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 7 Aug 2020 10:31:09 -0500 Subject: [PATCH 1/6] Include replicaN in /info response --- api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api.go b/api.go index 19a433315..501187512 100644 --- a/api.go +++ b/api.go @@ -1758,6 +1758,7 @@ func (api *API) Info() serverInfo { CPUType: si.CPUModel(), Memory: mem, TxSrc: api.holder.txf.TxType(), + ReplicaN: api.cluster.ReplicaN, } } @@ -2004,6 +2005,7 @@ func (api *API) TranslateFieldDB(ctx context.Context, indexName, fieldName strin type serverInfo struct { ShardWidth uint64 `json:"shardWidth"` + ReplicaN int `json:"replicaN"` Memory uint64 `json:"memory"` CPUType string `json:"cpuType"` CPUPhysicalCores int `json:"cpuPhysicalCores"` From 8fad2b92c2048ba2a336aaaa16e86901623ef93d Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Fri, 7 Aug 2020 11:14:24 -0500 Subject: [PATCH 2/6] Add all node states to /status response --- api.go | 6 ++++++ cluster.go | 6 ++++++ http/handler.go | 8 +++++--- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/api.go b/api.go index 501187512..4e11cc838 100644 --- a/api.go +++ b/api.go @@ -776,6 +776,12 @@ func (api *API) Hosts(ctx context.Context) []*Node { return api.cluster.Nodes() } +func (api *API) HostStates(ctx context.Context) map[string]string { + span, _ := tracing.StartSpanFromContext(ctx, "API.HostStates") + defer span.Finish() + return api.cluster.AllNodeStates() +} + // Node gets the ID, URI and coordinator status for this particular node. func (api *API) Node() *Node { node := api.server.node() diff --git a/cluster.go b/cluster.go index 6e5450cf4..1762be18d 100644 --- a/cluster.go +++ b/cluster.go @@ -700,6 +700,12 @@ func (c *cluster) Nodes() []*Node { return ret } +func (c *cluster) AllNodeStates() map[string]string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.Topology.nodeStates +} + // removeNodeBasicSorted removes a node from the cluster, maintaining the sort // order. Returns true if the node was removed. unprotected. func (c *cluster) removeNodeBasicSorted(nodeID string) bool { diff --git a/http/handler.go b/http/handler.go index f000a365c..186e857ea 100644 --- a/http/handler.go +++ b/http/handler.go @@ -702,6 +702,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { status := getStatusResponse{ State: h.api.State(), Nodes: h.api.Hosts(r.Context()), + NodeStates: h.api.HostStates(r.Context()), LocalID: h.api.Node().ID, } w.Header().Set("Content-Type", "application/json") @@ -758,9 +759,10 @@ type getSchemaResponse struct { } type getStatusResponse struct { - State string `json:"state"` - Nodes []*pilosa.Node `json:"nodes"` - LocalID string `json:"localID"` + State string `json:"state"` + Nodes []*pilosa.Node `json:"nodes"` + NodeStates map[string]string `json:"nodeStates"` + LocalID string `json:"localID"` } func hash(s string) string { From 5705b45864ba52256b1917f231a302a6545de58a Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Tue, 11 Aug 2020 10:57:59 -0500 Subject: [PATCH 3/6] Add a CLI flag for cluster name and include it in /status response --- api.go | 10 ++++++++++ cluster.go | 3 +++ ctl/server.go | 1 + http/handler.go | 20 ++++++++++++-------- server.go | 8 ++++++++ server/config.go | 1 + server/server.go | 1 + 7 files changed, 36 insertions(+), 8 deletions(-) diff --git a/api.go b/api.go index 4e11cc838..232a510e8 100644 --- a/api.go +++ b/api.go @@ -1739,6 +1739,16 @@ func (api *API) State() string { return api.cluster.State() } +// ClusterID returns the cluster ID. +func (api *API) ClusterID() string { + return api.cluster.id +} + +// ClusterName returns the cluster name. +func (api *API) ClusterName() string { + return api.cluster.Name +} + // Version returns the Pilosa version. func (api *API) Version() string { return strings.TrimPrefix(Version, "v") diff --git a/cluster.go b/cluster.go index 1762be18d..f31a819e5 100644 --- a/cluster.go +++ b/cluster.go @@ -208,6 +208,9 @@ type cluster struct { // nolint: maligned // The number of replicas a partition has. ReplicaN int + // Human-readable name of the cluster. + Name string + // Threshold for logging long-running queries // TODO(2.0) move this out of cluster. (why is it here??) longQueryTime time.Duration diff --git a/ctl/server.go b/ctl/server.go index 968b2bd6d..221f3918f 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -47,6 +47,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") + flags.StringVar(&srv.Config.Cluster.Name, "cluster.name", srv.Config.Cluster.Name, "Human-readable name for the cluster.") // Translation flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.") diff --git a/http/handler.go b/http/handler.go index 186e857ea..6f08664a4 100644 --- a/http/handler.go +++ b/http/handler.go @@ -700,10 +700,12 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { return } status := getStatusResponse{ - State: h.api.State(), - Nodes: h.api.Hosts(r.Context()), - NodeStates: h.api.HostStates(r.Context()), - LocalID: h.api.Node().ID, + State: h.api.State(), + Nodes: h.api.Hosts(r.Context()), + NodeStates: h.api.HostStates(r.Context()), + LocalID: h.api.Node().ID, + ClusterID: h.api.ClusterID(), + ClusterName: h.api.ClusterName(), } w.Header().Set("Content-Type", "application/json") if err := json.NewEncoder(w).Encode(status); err != nil { @@ -759,10 +761,12 @@ type getSchemaResponse struct { } type getStatusResponse struct { - State string `json:"state"` - Nodes []*pilosa.Node `json:"nodes"` - NodeStates map[string]string `json:"nodeStates"` - LocalID string `json:"localID"` + State string `json:"state"` + Nodes []*pilosa.Node `json:"nodes"` + NodeStates map[string]string `json:"nodeStates"` + LocalID string `json:"localID"` + ClusterID string `json:"clusterID"` + ClusterName string `json:"clusterName"` } func hash(s string) string { diff --git a/server.go b/server.go index a93df180a..a011ab0ba 100644 --- a/server.go +++ b/server.go @@ -269,6 +269,14 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { } } +// OptServerClusterName sets the human-readable cluster name. +func OptServerClusterName(name string) ServerOption { + return func(s *Server) error { + s.cluster.Name = name + return nil + } +} + // OptServerSerializer is a functional option on Server // used to set the serializer. func OptServerSerializer(ser Serializer) ServerOption { diff --git a/server/config.go b/server/config.go index c51a7f95f..3e8575c81 100644 --- a/server/config.go +++ b/server/config.go @@ -122,6 +122,7 @@ type Config struct { Coordinator bool `toml:"coordinator"` ReplicaN int `toml:"replicas"` Hosts []string `toml:"hosts"` + Name string `toml:"name"` // TODO(2.0) move this out of cluster. (why is it here??) LongQueryTime toml.Duration `toml:"long-query-time"` } `toml:"cluster"` diff --git a/server/server.go b/server/server.go index 1b7dea316..077c8a55c 100644 --- a/server/server.go +++ b/server/server.go @@ -406,6 +406,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerGRPCURI(advertiseGRPCURI), pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts), + pilosa.OptServerClusterName(m.Config.Cluster.Name), pilosa.OptServerSerializer(proto.Serializer{}), pilosa.OptServerTxsrc(m.Config.Txsrc), coordinatorOpt, From 1a04e94fc179d480a0578b19c1318c0170739fcb Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 20 Aug 2020 17:52:25 -0500 Subject: [PATCH 4/6] Remove clusterID and default name to ID --- api.go | 8 +++----- http/handler.go | 2 -- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/api.go b/api.go index 232a510e8..018eb8091 100644 --- a/api.go +++ b/api.go @@ -1739,13 +1739,11 @@ func (api *API) State() string { return api.cluster.State() } -// ClusterID returns the cluster ID. -func (api *API) ClusterID() string { - return api.cluster.id -} - // ClusterName returns the cluster name. func (api *API) ClusterName() string { + if api.cluster.Name == "" { + return api.cluster.id + } return api.cluster.Name } diff --git a/http/handler.go b/http/handler.go index 6f08664a4..63e51ef6d 100644 --- a/http/handler.go +++ b/http/handler.go @@ -704,7 +704,6 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { Nodes: h.api.Hosts(r.Context()), NodeStates: h.api.HostStates(r.Context()), LocalID: h.api.Node().ID, - ClusterID: h.api.ClusterID(), ClusterName: h.api.ClusterName(), } w.Header().Set("Content-Type", "application/json") @@ -765,7 +764,6 @@ type getStatusResponse struct { Nodes []*pilosa.Node `json:"nodes"` NodeStates map[string]string `json:"nodeStates"` LocalID string `json:"localID"` - ClusterID string `json:"clusterID"` ClusterName string `json:"clusterName"` } From 46d010fe188b8ed42643c6af464b2bdada2e247d Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Wed, 30 Sep 2020 07:23:39 -0500 Subject: [PATCH 5/6] Remove 'NodeStates' from /status until it is reliable --- http/handler.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/http/handler.go b/http/handler.go index 63e51ef6d..c91adef68 100644 --- a/http/handler.go +++ b/http/handler.go @@ -702,7 +702,6 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) { status := getStatusResponse{ State: h.api.State(), Nodes: h.api.Hosts(r.Context()), - NodeStates: h.api.HostStates(r.Context()), LocalID: h.api.Node().ID, ClusterName: h.api.ClusterName(), } @@ -760,11 +759,10 @@ type getSchemaResponse struct { } type getStatusResponse struct { - State string `json:"state"` - Nodes []*pilosa.Node `json:"nodes"` - NodeStates map[string]string `json:"nodeStates"` - LocalID string `json:"localID"` - ClusterName string `json:"clusterName"` + State string `json:"state"` + Nodes []*pilosa.Node `json:"nodes"` + LocalID string `json:"localID"` + ClusterName string `json:"clusterName"` } func hash(s string) string { From 8a3d93fe78847ae32f2a1684ccc1f0790d59cb77 Mon Sep 17 00:00:00 2001 From: Alan Bernstein Date: Thu, 1 Oct 2020 10:45:08 -0500 Subject: [PATCH 6/6] Move TransactionList endpoint to UI namespace --- http/handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http/handler.go b/http/handler.go index c91adef68..98634111d 100644 --- a/http/handler.go +++ b/http/handler.go @@ -381,8 +381,6 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET").Name("GetSchema") router.HandleFunc("/schema", handler.handlePostSchema).Methods("POST").Name("PostSchema") router.HandleFunc("/status", handler.handleGetStatus).Methods("GET").Name("GetStatus") - router.HandleFunc("/transaction", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList") - router.HandleFunc("/transaction/", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList") router.HandleFunc("/transaction", handler.handlePostTransaction).Methods("POST").Name("PostTransaction") router.HandleFunc("/transaction/", handler.handlePostTransaction).Methods("POST").Name("PostTransaction") router.HandleFunc("/transaction/{id}", handler.handleGetTransaction).Methods("GET").Name("GetTransaction") @@ -393,6 +391,8 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion") router.HandleFunc("/ui/usage", handler.handleGetUsage).Methods("GET").Name("GetUsage") + router.HandleFunc("/ui/transaction", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList") + router.HandleFunc("/ui/transaction/", handler.handleGetTransactionList).Methods("GET").Name("GetTransactionList") // /internal endpoints are for internal use only; they may change at any time. // DO NOT rely on these for external applications!