From 599b2f4a9e329e0a05459ee8a47b5ebfa23d35da Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Tue, 26 Feb 2019 16:58:06 -0600 Subject: [PATCH] implement config options for block profile rate and mutex fraction set sane defaults. The performance overhead seems to be negligible, and this will allow us to obtain mutex and blocking profiles from running Pilosas by default. --- cmd/server_test.go | 38 +++++++++++++++++++++++++++++++++++--- ctl/server.go | 4 ++++ docs/configuration.md | 26 ++++++++++++++++++++++++++ server/config.go | 10 ++++++++++ server/server.go | 4 ++++ 5 files changed, 79 insertions(+), 3 deletions(-) diff --git a/cmd/server_test.go b/cmd/server_test.go index 84b2297d4..291542f8d 100644 --- a/cmd/server_test.go +++ b/cmd/server_test.go @@ -43,7 +43,13 @@ func TestServerConfig(t *testing.T) { // TEST 0 { args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111", "--translation.map-size", "100000"}, - env: map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"}, + env: map[string]string{ + "PILOSA_DATA_DIR": "/tmp/myEnvDatadir", + "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", + "PILOSA_MAX_WRITES_PER_REQUEST": "2000", + "PILOSA_PROFILE_BLOCK_RATE": "9123", + "PILOSA_PROFILE_MUTEX_FRACTION": "444", + }, cfgFileContent: ` data-dir = "/tmp/myFileDatadir" bind = "localhost:0" @@ -56,6 +62,9 @@ func TestServerConfig(t *testing.T) { "localhost:19444", ] long-query-time = "1m10s" + [profile] + block-rate = 100 + mutex-fraction = 10 `, validation: func() error { v := validator{} @@ -66,13 +75,25 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90)) v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000) v.Check(cmd.Server.Config.Translation.MapSize, 100000) + v.Check(cmd.Server.Config.Profile.BlockRate, 9123) + v.Check(cmd.Server.Config.Profile.MutexFraction, 444) return v.Error() }, }, // TEST 1 { - args: []string{"server", "--anti-entropy.interval", "9m0s"}, - env: map[string]string{"PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", "PILOSA_BIND": "localhost:1110", "PILOSA_TRANSLATION_MAP_SIZE": "100000"}, + args: []string{"server", + "--anti-entropy.interval", "9m0s", + "--profile.block-rate", "4832", + "--profile.mutex-fraction", "8290", + }, + env: map[string]string{ + "PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", + "PILOSA_BIND": "localhost:1110", + "PILOSA_TRANSLATION_MAP_SIZE": "100000", + "PILOSA_PROFILE_BLOCK_RATE": "9123", + "PILOSA_PROFILE_MUTEX_FRACTION": "444", + }, cfgFileContent: ` bind = "localhost:0" data-dir = "` + actualDataDir + `" @@ -81,12 +102,17 @@ func TestServerConfig(t *testing.T) { hosts = [ "localhost:19444", ] + [profile] + block-rate = 100 + mutex-fraction = 10 `, validation: func() error { v := validator{} v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"}) v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9)) v.Check(cmd.Server.Config.Translation.MapSize, 100000) + v.Check(cmd.Server.Config.Profile.BlockRate, 4832) + v.Check(cmd.Server.Config.Profile.MutexFraction, 8290) return v.Error() }, }, @@ -106,6 +132,10 @@ func TestServerConfig(t *testing.T) { [metric] service = "statsd" host = "127.0.0.1:8125" + [profile] + block-rate = 100 + mutex-fraction = 10 + `, validation: func() error { v := validator{} @@ -114,6 +144,8 @@ func TestServerConfig(t *testing.T) { v.Check(cmd.Server.Config.LogPath, logFile.Name()) v.Check(cmd.Server.Config.Metric.Service, "statsd") v.Check(cmd.Server.Config.Metric.Host, "127.0.0.1:8125") + v.Check(cmd.Server.Config.Profile.BlockRate, 100) + v.Check(cmd.Server.Config.Profile.MutexFraction, 10) if v.Error() != nil { return v.Error() } diff --git a/ctl/server.go b/ctl/server.go index 2f727e9e0..ab4dd3305 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -79,4 +79,8 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.") flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type.") flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.") + + // Profiling + flags.IntVar(&srv.Config.Profile.BlockRate, "profile.block-rate", srv.Config.Profile.BlockRate, "Sampling rate for goroutine blocking profiler. One sample per ns.") + flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/ of events.") } diff --git a/docs/configuration.md b/docs/configuration.md index 2d046a961..e357e354b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -411,6 +411,32 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h agent-host-port = "localhost:6831" ``` +#### Profile Block Rate + +* Description: Block Rate is passed directly to Go's + runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1 + per `rate` nanoseconds. A value of "1" samples every event, and 0 disables + profiling. +* Flag: `--profile.block-rate=10000000` +* Env: `PILOSA_PROFILE_BLOCK_RATE=10000000` +* Config: + ```toml + [profile] + block-rate = 10000000 + ``` + +#### Profile Mutex Fraction + +* Description: Mutex Fraction is passed directly to Go's + runtime.SetMutexProfileFraction. 1/`fraction` of events will be sampled. +* Flag: `--profile.mutex-fraction=100` +* Env: `PILOSA_PROFILE_MUTEX_FRACTION=100` +* Config: + ```toml + [profile] + mutex-fraction = 100 + ``` + #### Translation Map Size * Description: Size in bytes of mmap to allocate for key translation diff --git a/server/config.go b/server/config.go index 8af776316..fc47f4017 100644 --- a/server/config.go +++ b/server/config.go @@ -128,6 +128,13 @@ type Config struct { // AgentHostPort is the host:port of the local agent. AgentHostPort string `toml:"agent-host-port"` } `toml:"tracing"` + + Profile struct { + // BlockRate is passed directly to runtime.SetBlockProfileRate + BlockRate int `toml:"block-rate` + // MutexFraction is passed directly to runtime.SetMutexProfileFraction + MutexFraction int `toml:"mutex-fraction"` + } `toml:"profile"` } // NewConfig returns an instance of Config with default options. @@ -170,6 +177,9 @@ func NewConfig() *Config { c.Tracing.SamplerType = jaeger.SamplerTypeRemote c.Tracing.SamplerParam = 0.001 + c.Profile.BlockRate = 10000000 // 1 sample per 10 ms + c.Profile.MutexFraction = 100 // 1% sampling + return c } diff --git a/server/server.go b/server/server.go index 868a5bda2..e376c5aa6 100644 --- a/server/server.go +++ b/server/server.go @@ -29,6 +29,7 @@ import ( "net" "os" "os/signal" + "runtime" "strconv" "syscall" "time" @@ -179,6 +180,9 @@ func (m *Command) Wait() error { // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { + runtime.SetBlockProfileRate(m.Config.Profile.BlockRate) + runtime.SetMutexProfileFraction(m.Config.Profile.MutexFraction) + syswrap.SetMaxMapCount(m.Config.MaxMapCount) syswrap.SetMaxFileCount(m.Config.MaxFileCount)