From 599b2f4a9e329e0a05459ee8a47b5ebfa23d35da Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 26 Feb 2019 16:58:06 -0600 Subject: [PATCH 1/4] implement config options for block profile rate and mutex fraction set sane defaults. The performance overhead seems to be negligible, and this will allow us to obtain mutex and blocking profiles from running Pilosas by default. --- cmd/server_test.go | 38 +++++++++++++++++++++++++++++++++++--- ctl/server.go | 4 ++++ docs/configuration.md | 26 ++++++++++++++++++++++++++ server/config.go | 10 ++++++++++ server/server.go | 4 ++++ 5 files changed, 79 insertions(+), 3 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index 84b2297d4..291542f8d 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -43,7 +43,13 @@ func TestServerConfig(t *testing.T) { // TEST 0 { args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111", "--translation.map-size", "100000"}, - env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, + env: map[string]string{ + "PILOSA_DATA_DIR": "/tmp/myEnvDatadir", + "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", + "PILOSA_MAX_WRITES_PER_REQUEST": "2000", + "PILOSA_PROFILE_BLOCK_RATE": "9123", + "PILOSA_PROFILE_MUTEX_FRACTION": "444", + }, cfgFileContent: ` data-dir = "/tmp/myFileDatadir" bind = "localhost:0" @@ -56,6 +62,9 @@ func TestServerConfig(t *testing.T) { "localhost:19444", ] long-query-time = "1m10s" + [profile] + block-rate = 100 + mutex-fraction = 10 `, validation: func() error { v := validator{} @@ -66,13 +75,25 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90)) v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000) v.Check(cmd.Server.Config.Translation.MapSize, 100000) + v.Check(cmd.Server.Config.Profile.BlockRate, 9123) + v.Check(cmd.Server.Config.Profile.MutexFraction, 444) return v.Error() }, }, // TEST 1 { - args: []string{"server", "--anti-entropy.interval", "9m0s"}, - env: map[string]string{"PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", "PILOSA_BIND": "localhost:1110", "PILOSA_TRANSLATION_MAP_SIZE": "100000"}, + args: []string{"server", + "--anti-entropy.interval", "9m0s", + "--profile.block-rate", "4832", + "--profile.mutex-fraction", "8290", + }, + env: map[string]string{ + "PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", + "PILOSA_BIND": "localhost:1110", + "PILOSA_TRANSLATION_MAP_SIZE": "100000", + "PILOSA_PROFILE_BLOCK_RATE": "9123", + "PILOSA_PROFILE_MUTEX_FRACTION": "444", + }, cfgFileContent: ` bind = "localhost:0" data-dir = "` + actualDataDir + `" @@ -81,12 +102,17 @@ func TestServerConfig(t *testing.T) { hosts = [ "localhost:19444", ] + [profile] + block-rate = 100 + mutex-fraction = 10 `, validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"}) v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9)) v.Check(cmd.Server.Config.Translation.MapSize, 100000) + v.Check(cmd.Server.Config.Profile.BlockRate, 4832) + v.Check(cmd.Server.Config.Profile.MutexFraction, 8290) return v.Error() }, }, @@ -106,6 +132,10 @@ func TestServerConfig(t *testing.T) { [metric] service = "statsd" host = "127.0.0.1:8125" + [profile] + block-rate = 100 + mutex-fraction = 10 + `, validation: func() error { v := validator{} @@ -114,6 +144,8 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.LogPath, logFile.Name()) v.Check(cmd.Server.Config.Metric.Service, "statsd") v.Check(cmd.Server.Config.Metric.Host, "127.0.0.1:8125") + v.Check(cmd.Server.Config.Profile.BlockRate, 100) + v.Check(cmd.Server.Config.Profile.MutexFraction, 10) if v.Error() != nil { return v.Error() } diff --git a/ctl/server.go b/ctl/server.go index 2f727e9e0..ab4dd3305 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -79,4 +79,8 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.") flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type.") flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.") + + // Profiling + flags.IntVar(&srv.Config.Profile.BlockRate, "profile.block-rate", srv.Config.Profile.BlockRate, "Sampling rate for goroutine blocking profiler. One sample per ns.") + flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/ of events.") } diff --git a/docs/configuration.md b/docs/configuration.md index 2d046a961..e357e354b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -411,6 +411,32 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h agent-host-port = "localhost:6831" ``` +#### Profile Block Rate + +* Description: Block Rate is passed directly to Go's + runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1 + per `rate` nanoseconds. A value of "1" samples every event, and 0 disables + profiling. +* Flag: `--profile.block-rate=10000000` +* Env: `PILOSA_PROFILE_BLOCK_RATE=10000000` +* Config: + ```toml + [profile] + block-rate = 10000000 + ``` + +#### Profile Mutex Fraction + +* Description: Mutex Fraction is passed directly to Go's + runtime.SetMutexProfileFraction. 1/`fraction` of events will be sampled. +* Flag: `--profile.mutex-fraction=100` +* Env: `PILOSA_PROFILE_MUTEX_FRACTION=100` +* Config: + ```toml + [profile] + mutex-fraction = 100 + ``` + #### Translation Map Size * Description: Size in bytes of mmap to allocate for key translation diff --git a/server/config.go b/server/config.go index 8af776316..fc47f4017 100644 --- a/server/config.go +++ b/server/config.go @@ -128,6 +128,13 @@ type Config struct { // AgentHostPort is the host:port of the local agent. AgentHostPort string `toml:"agent-host-port"` } `toml:"tracing"` + + Profile struct { + // BlockRate is passed directly to runtime.SetBlockProfileRate + BlockRate int `toml:"block-rate` + // MutexFraction is passed directly to runtime.SetMutexProfileFraction + MutexFraction int `toml:"mutex-fraction"` + } `toml:"profile"` } // NewConfig returns an instance of Config with default options. @@ -170,6 +177,9 @@ func NewConfig() *Config { c.Tracing.SamplerType = jaeger.SamplerTypeRemote c.Tracing.SamplerParam = 0.001 + c.Profile.BlockRate = 10000000 // 1 sample per 10 ms + c.Profile.MutexFraction = 100 // 1% sampling + return c } diff --git a/server/server.go b/server/server.go index 868a5bda2..e376c5aa6 100644 --- a/server/server.go +++ b/server/server.go @@ -29,6 +29,7 @@ import ( "net" "os" "os/signal" + "runtime" "strconv" "syscall" "time" @@ -179,6 +180,9 @@ func (m *Command) Wait() error { // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { + runtime.SetBlockProfileRate(m.Config.Profile.BlockRate) + runtime.SetMutexProfileFraction(m.Config.Profile.MutexFraction) + syswrap.SetMaxMapCount(m.Config.MaxMapCount) syswrap.SetMaxFileCount(m.Config.MaxFileCount) From 9f4a9421bcbd1ccd045a640e82adc44039261af9 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 25 Mar 2019 12:16:24 -0500 Subject: [PATCH 2/4] fix missing quote in toml tag. unclear how test could pass without it --- cmd/server_test.go | 8 ++++---- server/config.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index 291542f8d..3efabe741 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -133,8 +133,8 @@ func TestServerConfig(t *testing.T) { service = "statsd" host = "127.0.0.1:8125" [profile] - block-rate = 100 - mutex-fraction = 10 + block-rate = 5352 + mutex-fraction = 91 `, validation: func() error { @@ -144,8 +144,8 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.LogPath, logFile.Name()) v.Check(cmd.Server.Config.Metric.Service, "statsd") v.Check(cmd.Server.Config.Metric.Host, "127.0.0.1:8125") - v.Check(cmd.Server.Config.Profile.BlockRate, 100) - v.Check(cmd.Server.Config.Profile.MutexFraction, 10) + v.Check(cmd.Server.Config.Profile.BlockRate, 5352) + v.Check(cmd.Server.Config.Profile.MutexFraction, 91) if v.Error() != nil { return v.Error() } diff --git a/server/config.go b/server/config.go index fc47f4017..9a7147307 100644 --- a/server/config.go +++ b/server/config.go @@ -131,7 +131,7 @@ type Config struct { Profile struct { // BlockRate is passed directly to runtime.SetBlockProfileRate - BlockRate int `toml:"block-rate` + BlockRate int `toml:"block-rate"` // MutexFraction is passed directly to runtime.SetMutexProfileFraction MutexFraction int `toml:"mutex-fraction"` } `toml:"profile"` From 54a6e0ef8465965123a004ed3c399ef1b5fadf25 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 25 Mar 2019 13:45:28 -0500 Subject: [PATCH 3/4] linkify SetBlockProfileRate in docs Co-Authored-By: jaffee --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index e357e354b..190cb0731 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -414,7 +414,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h #### Profile Block Rate * Description: Block Rate is passed directly to Go's - runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1 + [runtime.SetBlockProfileRate](https://golang.org/pkg/runtime/#SetBlockProfileRate). Goroutine blocking events will be sampled at 1 per `rate` nanoseconds. A value of "1" samples every event, and 0 disables profiling. * Flag: `--profile.block-rate=10000000` From 52062c0a270dc8ed24e01f7663f7819d7e0a9acb Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Mon, 25 Mar 2019 13:45:43 -0500 Subject: [PATCH 4/4] linkify SetMutexProfileFraction in docs Co-Authored-By: jaffee --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 190cb0731..5a0313d63 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -428,7 +428,7 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h #### Profile Mutex Fraction * Description: Mutex Fraction is passed directly to Go's - runtime.SetMutexProfileFraction. 1/`fraction` of events will be sampled. + [runtime.SetMutexProfileFraction](https://golang.org/pkg/runtime/#SetMutexProfileFraction). 1/`fraction` of events will be sampled. * Flag: `--profile.mutex-fraction=100` * Env: `PILOSA_PROFILE_MUTEX_FRACTION=100` * Config: