From d2d0756f4a22db4387796e22c671da2c4b04747d Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Fri, 15 Dec 2017 14:39:35 -0600 Subject: [PATCH] Define default config values for pilosa server in NewConfig rather than flags. Based on work done in #885. --- cmd/server_test.go | 3 +-- config.go | 66 ++++++++++++++++++++++++++++++---------------- ctl/config_test.go | 7 ++--- ctl/server.go | 60 +++++++++++++++++++++++------------------ server.go | 1 - server/server.go | 3 --- 6 files changed, 82 insertions(+), 58 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index b650e316a..c9b900bde 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -45,7 +45,7 @@ func TestServerConfig(t *testing.T) { // TEST 0 { args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111"}, - env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_POLL_INTERVAL": "3m2s", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, + env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, cfgFileContent: ` data-dir = "/tmp/myFileDatadir" bind = "localhost:0" @@ -65,7 +65,6 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Bind, "localhost:10111") v.Check(cmd.Server.Config.Cluster.ReplicaN, 2) v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:10111", "localhost:10110"}) - v.Check(cmd.Server.Config.Cluster.PollInterval, pilosa.Duration(time.Second*182)) v.Check(cmd.Server.Config.Cluster.LongQueryTime, pilosa.Duration(time.Second*90)) v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000) return v.Error() diff --git a/config.go b/config.go index 9dfbb53ac..a43963f93 100644 --- a/config.go +++ b/config.go @@ -26,6 +26,9 @@ const ( ) const ( + // DefaultDataDir is the default data directory. + DefaultDataDir = "~/.pilosa" + // DefaultHost is the default hostname to use. DefaultHost = "localhost" @@ -106,6 +109,8 @@ const ( DefaultGossipGossipInterval = 200 * time.Millisecond DefaultGossipGossipNodes = 3 DefaultGossipGossipToTheDeadTime = 30 * time.Second + + DefaultMetricPollInterval = 0 * time.Minute ) // ClusterTypes set of cluster types. @@ -130,6 +135,23 @@ type Config struct { // GossipSeed DEPRECATED GossipSeed string `toml:"gossip-seed"` + // Limits the number of mutating commands that can be in a single request to + // the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs. + MaxWritesPerRequest int `toml:"max-writes-per-request"` + + LogPath string `toml:"log-path"` + + // TLS + TLS TLSConfig + + Cluster struct { + Coordinator string `toml:"coordinator"` + ReplicaN int `toml:"replicas"` + Type string `toml:"type"` + Hosts []string `toml:"hosts"` + LongQueryTime Duration `toml:"long-query-time"` + } `toml:"cluster"` + Gossip struct { Port string `toml:"port"` Seed string `toml:"seed"` @@ -144,50 +166,39 @@ type Config struct { GossipToTheDeadTime Duration `toml:"gossip-to-the-dead-time"` } `toml:"gossip"` - Cluster struct { - Coordinator string `toml:"coordinator"` - ReplicaN int `toml:"replicas"` - Type string `toml:"type"` - Hosts []string `toml:"hosts"` - PollInterval Duration `toml:"poll-interval"` - LongQueryTime Duration `toml:"long-query-time"` - } `toml:"cluster"` - AntiEntropy struct { Interval Duration `toml:"interval"` } `toml:"anti-entropy"` - // Limits the number of mutating commands that can be in a single request to - // the server. This includes SetBit, ClearBit, SetRowAttrs & SetColumnAttrs. - MaxWritesPerRequest int `toml:"max-writes-per-request"` - - LogPath string `toml:"log-path"` - Metric struct { Service string `toml:"service"` Host string `toml:"host"` PollInterval Duration `toml:"poll-interval"` Diagnostics bool `toml:"diagnostics"` } `toml:"metric"` - - TLS TLSConfig } // NewConfig returns an instance of Config with default options. func NewConfig() *Config { c := &Config{ - Bind: DefaultHost + ":" + DefaultPort, + DataDir: DefaultDataDir, + Bind: ":" + DefaultPort, MaxWritesPerRequest: DefaultMaxWritesPerRequest, + // LogPath: "", + TLS: TLSConfig{}, } + + // Cluster config. + // c.Cluster.Coordinator = "" c.Cluster.ReplicaN = DefaultReplicaN c.Cluster.Type = DefaultClusterType c.Cluster.Hosts = []string{} - c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) - c.Metric.Service = DefaultMetrics - c.Metric.Diagnostics = true - c.TLS = TLSConfig{} + c.Cluster.LongQueryTime = Duration(time.Minute) - // Gossip related config. + // Gossip config. + // c.Gossip.Port = "" + // c.Gossip.Seed = "" + // c.Gossip.Key = "" c.Gossip.StreamTimeout = Duration(DefaultGossipStreamTimeout) c.Gossip.SuspicionMult = DefaultGossipSuspicionMult c.Gossip.PushPullInterval = Duration(DefaultGossipPushPullInterval) @@ -197,6 +208,15 @@ func NewConfig() *Config { c.Gossip.GossipInterval = Duration(DefaultGossipGossipInterval) c.Gossip.GossipToTheDeadTime = Duration(DefaultGossipGossipToTheDeadTime) + // AntiEntropy config. + c.AntiEntropy.Interval = Duration(DefaultAntiEntropyInterval) + + // Metric config. + c.Metric.Service = DefaultMetrics + // c.Metric.Host = "" + c.Metric.PollInterval = Duration(DefaultMetricPollInterval) + c.Metric.Diagnostics = true + return c } diff --git a/ctl/config_test.go b/ctl/config_test.go index a3bd035ac..b54446e34 100644 --- a/ctl/config_test.go +++ b/ctl/config_test.go @@ -17,11 +17,12 @@ package ctl import ( "bytes" "context" - "github.com/pilosa/pilosa" "io" "os" "strings" "testing" + + "github.com/pilosa/pilosa" ) func TestConfigCommand_Run(t *testing.T) { @@ -38,7 +39,7 @@ func TestConfigCommand_Run(t *testing.T) { if err != nil { t.Fatalf("Config Run doesn't work: %s", err) - } else if !strings.Contains(buf.String(), pilosa.DefaultHost) { - t.Fatalf("Unexpected config: %s", buf.String()) + } else if !strings.Contains(buf.String(), ":10101") { + t.Fatalf("Unexpected config: \n%s", buf.String()) } } diff --git a/ctl/server.go b/ctl/server.go index ba49aded1..1dddeb0d7 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -24,38 +24,46 @@ import ( // BuildServerFlags attaches a set of flags to the command for a server instance. func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags := cmd.Flags() - flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", "~/.pilosa", "Directory to store pilosa data files.") - flags.StringVarP(&srv.Config.Bind, "bind", "b", ":10101", "Default URI on which pilosa should listen.") + flags.StringVarP(&srv.Config.DataDir, "data-dir", "d", srv.Config.DataDir, "Directory to store pilosa data files.") + flags.StringVarP(&srv.Config.Bind, "bind", "b", srv.Config.Bind, "Default URI on which pilosa should listen.") flags.StringVarP(&srv.Config.GossipPort, "gossip-port", "", "", "(DEPRECATED) Port to which pilosa should bind for internal state sharing.") flags.StringVarP(&srv.Config.GossipSeed, "gossip-seed", "", "", "(DEPRECATED) Host with which to seed the gossip membership.") - // gossip - flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", "", "Port to which pilosa should bind for internal state sharing.") - flags.StringVarP(&srv.Config.Gossip.Seed, "gossip.seed", "", "", "Host with which to seed the gossip membership.") - flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", "", "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.") - - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", 10*time.Second, "Timeout for establishing a stream connection with a remote node for a full state sync.") - flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", 4, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.") - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", 30*time.Second, "Interval between complete state syncs.") - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", 500*time.Millisecond, "Timeout to wait for an ack from a probed node before assuming it is unhealthy.") - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", 1*time.Second, "Interval between random node probes.") - flags.IntVarP(&srv.Config.Gossip.GossipNodes, "gossip.gossip-nodes", "", 3, "Number of random nodes to send gossip messages to per GossipInterval.") - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.GossipInterval), "gossip.gossip-interval", "", 200*time.Millisecond, "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.") - flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.GossipToTheDeadTime), "gossip.gossip-to-the-dead-time", "", 30*time.Second, "Interval after which a node has died that we will still try to gossip to it.") - - flags.StringVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", "", "Host that will act as cluster coordinator during startup and resizing.") flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") + flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path") + + // TLS + SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) + + // Cluster + flags.StringVarP(&srv.Config.Cluster.Coordinator, "cluster.coordinator", "", "", "Host that will act as cluster coordinator during startup and resizing.") flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") + flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster.") - flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.PollInterval), "cluster.poll-interval", "", time.Minute, "Polling interval for cluster.") // TODO what actually is this? flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") - flags.StringVar(&srv.Config.LogPath, "log-path", "", "Log path") - flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", time.Minute*10, "Interval at which to run anti-entropy routine.") + + // Gossip + flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") + flags.StringVarP(&srv.Config.Gossip.Seed, "gossip.seed", "", srv.Config.Gossip.Seed, "Host with which to seed the gossip membership.") + flags.StringVarP(&srv.Config.Gossip.Key, "gossip.key", "", srv.Config.Gossip.Key, "The path to file of the encryption key for gossip. The contents of the file should be either 16, 24, or 32 bytes to select AES-128, AES-192, or AES-256.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.StreamTimeout), "gossip.stream-timeout", "", (time.Duration)(srv.Config.Gossip.StreamTimeout), "Timeout for establishing a stream connection with a remote node for a full state sync.") + flags.IntVarP(&srv.Config.Gossip.SuspicionMult, "gossip.suspicion-mult", "", srv.Config.Gossip.SuspicionMult, "Multiplier for determining the time an inaccessible node is considered suspect before declaring it dead.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.PushPullInterval), "gossip.push-pull-interval", "", (time.Duration)(srv.Config.Gossip.PushPullInterval), "Interval between complete state syncs.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeTimeout), "gossip.probe-timeout", "", (time.Duration)(srv.Config.Gossip.ProbeTimeout), "Timeout to wait for an ack from a probed node before assuming it is unhealthy.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.ProbeInterval), "gossip.probe-interval", "", (time.Duration)(srv.Config.Gossip.ProbeInterval), "Interval between random node probes.") + flags.IntVarP(&srv.Config.Gossip.GossipNodes, "gossip.gossip-nodes", "", srv.Config.Gossip.GossipNodes, "Number of random nodes to send gossip messages to per GossipInterval.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.GossipInterval), "gossip.gossip-interval", "", (time.Duration)(srv.Config.Gossip.GossipInterval), "Interval between sending messages that need to be gossiped that haven't piggybacked on probing messages.") + flags.DurationVarP((*time.Duration)(&srv.Config.Gossip.GossipToTheDeadTime), "gossip.gossip-to-the-dead-time", "", (time.Duration)(srv.Config.Gossip.GossipToTheDeadTime), "Interval after which a node has died that we will still try to gossip to it.") + + // AntiEntropy + flags.DurationVarP((*time.Duration)(&srv.Config.AntiEntropy.Interval), "anti-entropy.interval", "", (time.Duration)(srv.Config.AntiEntropy.Interval), "Interval at which to run anti-entropy routine.") + + // Metric + flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", srv.Config.Metric.Service, "Default URI on which pilosa should listen.") + flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", srv.Config.Metric.Host, "Default URI to send metrics.") + flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", (time.Duration)(srv.Config.Metric.PollInterval), "Polling interval metrics.") + flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", srv.Config.Metric.Diagnostics, "Enabled diagnostics reporting.") + + // CPU Profiling flags.StringVarP(&srv.CPUProfile, "profile.cpu", "", "", "Where to store CPU profile.") flags.DurationVarP(&srv.CPUTime, "profile.cpu-time", "", 30*time.Second, "CPU profile duration.") - flags.StringVarP(&srv.Config.Cluster.Type, "cluster.type", "", "gossip", "Determine how the cluster handles membership and state sharing. Choose from [static, gossip]") - flags.StringVarP(&srv.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.") - flags.StringVarP(&srv.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.") - flags.BoolVarP((&srv.Config.Metric.Diagnostics), "metric.diagnostics", "", true, "Enabled diagnostics reporting.") - flags.DurationVarP((*time.Duration)(&srv.Config.Metric.PollInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.") - SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) } diff --git a/server.go b/server.go index 1d03bd78b..cbeeb53a1 100644 --- a/server.go +++ b/server.go @@ -42,7 +42,6 @@ import ( // Default server settings. const ( DefaultAntiEntropyInterval = 10 * time.Minute - DefaultPollingInterval = 60 * time.Second DefaultDiagnosticServer = "https://diagnostics.pilosa.com/v0/diagnostics" ) diff --git a/server/server.go b/server/server.go index 27c3c2e4c..d238d1c2f 100644 --- a/server/server.go +++ b/server/server.go @@ -41,9 +41,6 @@ func init() { } const ( - // DefaultDataDir is the default data directory. - DefaultDataDir = "~/.pilosa" - // DefaultDiagnosticsInterval is the default sync frequency diagnostic metrics. DefaultDiagnosticsInterval = 1 * time.Hour )