From 4b53ec1f9c05653c136e957429cd05a9a780f285 Mon Sep 17 00:00:00 2001 From: Michael Baird Date: Tue, 11 Jul 2017 14:17:06 -0500 Subject: [PATCH] const cluster types, and more validation tests --- cmd/server_test.go | 10 ++++++++++ config.go | 46 ++++++++++++++++++++++++++++++++++------------ config_test.go | 42 ++++++++++++++++++++++++++++++++++++++---- pilosa.go | 9 +++++++-- server/server.go | 6 +++--- 5 files changed, 92 insertions(+), 21 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index a0f6096c8..25d0d7fd1 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -52,10 +52,15 @@ func TestServerConfig(t *testing.T) { [cluster] poll-interval = "45s" + type = "http" replicas = 2 hosts = [ "localhost:19444", ] + internal-hosts = [ + "localhost:19500", + "localhost:19501", + ] `, validation: func() error { v := validator{} @@ -75,9 +80,14 @@ func TestServerConfig(t *testing.T) { bind = "localhost:0" data-dir = "` + actualDataDir + `" [cluster] + type = "http" hosts = [ "localhost:19444", ] + internal-hosts = [ + "localhost:19500", + "localhost:19501", + ] [plugins] path = "/var/sloth" `, diff --git a/config.go b/config.go index b926a44fa..8607c3c86 100644 --- a/config.go +++ b/config.go @@ -14,7 +14,17 @@ package pilosa -import "time" +import ( + "time" +) + +// Cluster types. +const ( + ClusterNone = "" + ClusterStatic = "static" + ClusterHTTP = "http" + ClusterGossip = "gossip" +) const ( // DefaultHost is the default hostname to use. @@ -24,7 +34,7 @@ const ( DefaultPort = "10101" // DefaultClusterType sets the node intercommunication method. - DefaultClusterType = "static" + DefaultClusterType = ClusterStatic // DefaultInternalPort the port the nodes intercommunicate on. DefaultInternalPort = "14000" @@ -36,6 +46,9 @@ const ( DefaultMaxWritesPerRequest = 5000 ) +// ClusterTypes set of cluster types. +var ClusterTypes = []string{ClusterNone, ClusterStatic, ClusterHTTP, ClusterGossip} + // Config represents the configuration for the command. type Config struct { DataDir string `toml:"data-dir"` @@ -91,21 +104,30 @@ func NewConfig() *Config { // Validate that all configuration permutations are compatible with each other. func (c *Config) Validate() error { - if !foundItem(c.Cluster.Hosts, c.Bind) { - return ErrConfigHosts + if !StringInSlice(c.Cluster.Type, ClusterTypes) { + return ErrConfigClusterTypeInvalid } - - // Validate cluster types - // TODO cluster types - // TODO validate len hosts - // TODO vaidate replica num and host len - // TODO internal-hosts and hosts len must match - - if c.Cluster.Type == "http" || c.Cluster.Type == "gossip" { + if len(c.Cluster.Hosts) > 1 && !(c.Cluster.Type == ClusterHTTP || c.Cluster.Type == ClusterGossip) { + return ErrConfigClusterTypeMissing + } + if c.Cluster.Type == ClusterHTTP || c.Cluster.Type == ClusterGossip { + if c.Cluster.ReplicaN > len(c.Cluster.Hosts) { + return ErrConfigReplicaNInvalid + } + if len(c.Cluster.Hosts) != len(c.Cluster.InternalHosts) { + return ErrConfigHostsMismatch + } + if !foundItem(c.Cluster.Hosts, c.Bind) { + return ErrConfigHostsMissing + } if !ContainsSubstring(c.Cluster.InternalPort, c.Cluster.InternalHosts) { return ErrConfigBroadcastPort } } + if c.Cluster.Type == ClusterGossip && !StringInSlice(c.Cluster.GossipSeed, c.Cluster.InternalHosts) { + return ErrConfigGossipSeed + } + return nil } diff --git a/config_test.go b/config_test.go index 1109f1436..9f7767a4e 100644 --- a/config_test.go +++ b/config_test.go @@ -7,12 +7,46 @@ import ( ) func Test_NewConfig(t *testing.T) { - x := pilosa.NewConfig() + c := pilosa.NewConfig() - // Check for bind addres in cluster hosts - if err := x.Validate(); err != pilosa.ErrConfigHosts { + c.Cluster.Hosts = []string{c.Bind, "localhost:10102"} + if err := c.Validate(); err != pilosa.ErrConfigClusterTypeMissing { t.Fatal(err) } - x.Cluster.Type = "http" + c.Cluster.Type = "test" + if err := c.Validate(); err != pilosa.ErrConfigClusterTypeInvalid { + t.Fatal(err) + } + + c.Cluster.Type = pilosa.ClusterHTTP + if err := c.Validate(); err != pilosa.ErrConfigHostsMismatch { + t.Fatal(err) + } + + c.Cluster.InternalPort = pilosa.DefaultInternalPort + c.Cluster.InternalHosts = []string{"localhost:14004", "localhost:14001"} + if err := c.Validate(); err != pilosa.ErrConfigBroadcastPort { + t.Fatal(err) + } + + c.Cluster.InternalHosts = []string{"localhost:14000", "localhost:14001"} + c.Bind = "localhost:1" + // Check for bind addres in cluster hosts + if err := c.Validate(); err != pilosa.ErrConfigHostsMissing { + t.Fatal(err) + } + + c.Bind = "localhost:10101" + c.Cluster.ReplicaN = 3 + if err := c.Validate(); err != pilosa.ErrConfigReplicaNInvalid { + t.Fatal(err) + } + + c.Cluster.ReplicaN = 2 + c.Cluster.Type = pilosa.ClusterGossip + c.Cluster.GossipSeed = "localhost:10101" + if err := c.Validate(); err != pilosa.ErrConfigGossipSeed { + t.Fatal(err) + } } diff --git a/pilosa.go b/pilosa.go index 8768541ef..51910ffee 100644 --- a/pilosa.go +++ b/pilosa.go @@ -58,8 +58,13 @@ var ( ErrQueryRequired = errors.New("query required") ErrTooManyWrites = errors.New("too many write commands") - ErrConfigHosts = errors.New("missing bind address in cluster hosts") - ErrConfigBroadcastPort = errors.New("missing broadcast port in internal-hosts") + ErrConfigClusterTypeInvalid = errors.New("invalid cluster type") + ErrConfigClusterTypeMissing = errors.New("missing cluster type") + ErrConfigHostsMissing = errors.New("missing bind address in cluster hosts") + ErrConfigBroadcastPort = errors.New("missing broadcast port in internal-hosts") + ErrConfigHostsMismatch = errors.New("hosts and internal-hosts length mismatch") + ErrConfigReplicaNInvalid = errors.New("replica number must be <= hosts") + ErrConfigGossipSeed = errors.New("invalid gossip seed") ) // Regular expression to validate index and frame names. diff --git a/server/server.go b/server/server.go index 3133dac05..6e51c4e74 100644 --- a/server/server.go +++ b/server/server.go @@ -158,7 +158,7 @@ func (m *Command) SetupServer() error { } switch m.Config.Cluster.Type { - case "http": + case pilosa.ClusterHTTP: m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, internalPortStr) m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(internalPortStr, m.Server.LogOutput) m.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet() @@ -166,7 +166,7 @@ func (m *Command) SetupServer() error { if err != nil { return err } - case "gossip": + case pilosa.ClusterGossip: gossipPort, err := strconv.Atoi(internalPortStr) if err != nil { return err @@ -184,7 +184,7 @@ func (m *Command) SetupServer() error { m.Server.Cluster.NodeSet = gossipNodeSet m.Server.Broadcaster = gossipNodeSet m.Server.BroadcastReceiver = gossipNodeSet - case "static", "": + case pilosa.ClusterStatic, pilosa.ClusterNone: m.Server.Broadcaster = pilosa.NopBroadcaster m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet() m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver