diff --git a/cmd/server_test.go b/cmd/server_test.go index 84b2297d4..291542f8d 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -43,7 +43,13 @@ func TestServerConfig(t *testing.T) { // TEST 0 { args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111", "--translation.map-size", "100000"}, - env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, + env: map[string]string{ + "PILOSA_DATA_DIR": "/tmp/myEnvDatadir", + "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", + "PILOSA_MAX_WRITES_PER_REQUEST": "2000", + "PILOSA_PROFILE_BLOCK_RATE": "9123", + "PILOSA_PROFILE_MUTEX_FRACTION": "444", + }, cfgFileContent: ` data-dir = "/tmp/myFileDatadir" bind = "localhost:0" @@ -56,6 +62,9 @@ func TestServerConfig(t *testing.T) { "localhost:19444", ] long-query-time = "1m10s" + [profile] + block-rate = 100 + mutex-fraction = 10 `, validation: func() error { v := validator{} @@ -66,13 +75,25 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90)) v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000) v.Check(cmd.Server.Config.Translation.MapSize, 100000) + v.Check(cmd.Server.Config.Profile.BlockRate, 9123) + v.Check(cmd.Server.Config.Profile.MutexFraction, 444) return v.Error() }, }, // TEST 1 { - args: []string{"server", "--anti-entropy.interval", "9m0s"}, - env: map[string]string{"PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", "PILOSA_BIND": "localhost:1110", "PILOSA_TRANSLATION_MAP_SIZE": "100000"}, + args: []string{"server", + "--anti-entropy.interval", "9m0s", + "--profile.block-rate", "4832", + "--profile.mutex-fraction", "8290", + }, + env: map[string]string{ + "PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", + "PILOSA_BIND": "localhost:1110", + "PILOSA_TRANSLATION_MAP_SIZE": "100000", + "PILOSA_PROFILE_BLOCK_RATE": "9123", + "PILOSA_PROFILE_MUTEX_FRACTION": "444", + }, cfgFileContent: ` bind = "localhost:0" data-dir = "` + actualDataDir + `" @@ -81,12 +102,17 @@ func TestServerConfig(t *testing.T) { hosts = [ "localhost:19444", ] + [profile] + block-rate = 100 + mutex-fraction = 10 `, validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"}) v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9)) v.Check(cmd.Server.Config.Translation.MapSize, 100000) + v.Check(cmd.Server.Config.Profile.BlockRate, 4832) + v.Check(cmd.Server.Config.Profile.MutexFraction, 8290) return v.Error() }, }, @@ -106,6 +132,10 @@ func TestServerConfig(t *testing.T) { [metric] service = "statsd" host = "127.0.0.1:8125" + [profile] + block-rate = 100 + mutex-fraction = 10 + `, validation: func() error { v := validator{} @@ -114,6 +144,8 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.LogPath, logFile.Name()) v.Check(cmd.Server.Config.Metric.Service, "statsd") v.Check(cmd.Server.Config.Metric.Host, "127.0.0.1:8125") + v.Check(cmd.Server.Config.Profile.BlockRate, 100) + v.Check(cmd.Server.Config.Profile.MutexFraction, 10) if v.Error() != nil { return v.Error() } diff --git a/ctl/server.go b/ctl/server.go index 2f727e9e0..ab4dd3305 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -79,4 +79,8 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.") flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type.") flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.") + + // Profiling + flags.IntVar(&srv.Config.Profile.BlockRate, "profile.block-rate", srv.Config.Profile.BlockRate, "Sampling rate for goroutine blocking profiler. One sample per ns.") + flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/ of events.") } diff --git a/docs/configuration.md b/docs/configuration.md index 2d046a961..e357e354b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -411,6 +411,32 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h agent-host-port = "localhost:6831" ``` +#### Profile Block Rate + +* Description: Block Rate is passed directly to Go's + runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1 + per `rate` nanoseconds. A value of "1" samples every event, and 0 disables + profiling. +* Flag: `--profile.block-rate=10000000` +* Env: `PILOSA_PROFILE_BLOCK_RATE=10000000` +* Config: + ```toml + [profile] + block-rate = 10000000 + ``` + +#### Profile Mutex Fraction + +* Description: Mutex Fraction is passed directly to Go's + runtime.SetMutexProfileFraction. 1/`fraction` of events will be sampled. +* Flag: `--profile.mutex-fraction=100` +* Env: `PILOSA_PROFILE_MUTEX_FRACTION=100` +* Config: + ```toml + [profile] + mutex-fraction = 100 + ``` + #### Translation Map Size * Description: Size in bytes of mmap to allocate for key translation diff --git a/server/config.go b/server/config.go index 8af776316..fc47f4017 100644 --- a/server/config.go +++ b/server/config.go @@ -128,6 +128,13 @@ type Config struct { // AgentHostPort is the host:port of the local agent. AgentHostPort string `toml:"agent-host-port"` } `toml:"tracing"` + + Profile struct { + // BlockRate is passed directly to runtime.SetBlockProfileRate + BlockRate int `toml:"block-rate` + // MutexFraction is passed directly to runtime.SetMutexProfileFraction + MutexFraction int `toml:"mutex-fraction"` + } `toml:"profile"` } // NewConfig returns an instance of Config with default options. @@ -170,6 +177,9 @@ func NewConfig() *Config { c.Tracing.SamplerType = jaeger.SamplerTypeRemote c.Tracing.SamplerParam = 0.001 + c.Profile.BlockRate = 10000000 // 1 sample per 10 ms + c.Profile.MutexFraction = 100 // 1% sampling + return c } diff --git a/server/server.go b/server/server.go index 868a5bda2..e376c5aa6 100644 --- a/server/server.go +++ b/server/server.go @@ -29,6 +29,7 @@ import ( "net" "os" "os/signal" + "runtime" "strconv" "syscall" "time" @@ -179,6 +180,9 @@ func (m *Command) Wait() error { // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { + runtime.SetBlockProfileRate(m.Config.Profile.BlockRate) + runtime.SetMutexProfileFraction(m.Config.Profile.MutexFraction) + syswrap.SetMaxMapCount(m.Config.MaxMapCount) syswrap.SetMaxFileCount(m.Config.MaxFileCount)