implement config options for block profile rate and mutex fraction

set sane defaults. The performance overhead seems to be negligible, and this will allow us to obtain mutex and blocking profiles from running Pilosas by default.

cmd/server_test.go

@@ -43,7 +43,13 @@ func TestServerConfig(t *testing.T) {
 		// TEST 0
 		{
 			args: []string{"server", "--data-dir", actualDataDir, "--cluster.hosts", "localhost:10111,localhost:10110", "--bind", "localhost:10111", "--translation.map-size", "100000"},
-			env:  map[string]string{"PILOSA_DATA_DIR": "/tmp/myEnvDatadir", "PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s", "PILOSA_MAX_WRITES_PER_REQUEST": "2000"},
+			env: map[string]string{
+				"PILOSA_DATA_DIR":                "/tmp/myEnvDatadir",
+				"PILOSA_CLUSTER_LONG_QUERY_TIME": "1m30s",
+				"PILOSA_MAX_WRITES_PER_REQUEST":  "2000",
+				"PILOSA_PROFILE_BLOCK_RATE":      "9123",
+				"PILOSA_PROFILE_MUTEX_FRACTION":  "444",
+			},
 			cfgFileContent: `
 	data-dir = "/tmp/myFileDatadir"
 	bind = "localhost:0"
@@ -56,6 +62,9 @@ func TestServerConfig(t *testing.T) {
 			"localhost:19444",
 		]
 		long-query-time = "1m10s"
+	[profile]
+		block-rate = 100
+		mutex-fraction = 10
 	`,
 			validation: func() error {
 				v := validator{}
@@ -66,13 +75,25 @@ func TestServerConfig(t *testing.T) {
 				v.Check(cmd.Server.Config.Cluster.LongQueryTime, toml.Duration(time.Second*90))
 				v.Check(cmd.Server.Config.MaxWritesPerRequest, 2000)
 				v.Check(cmd.Server.Config.Translation.MapSize, 100000)
+				v.Check(cmd.Server.Config.Profile.BlockRate, 9123)
+				v.Check(cmd.Server.Config.Profile.MutexFraction, 444)
 				return v.Error()
 			},
 		},
 		// TEST 1
 		{
-			args: []string{"server", "--anti-entropy.interval", "9m0s"},
-			env:  map[string]string{"PILOSA_CLUSTER_HOSTS": "localhost:1110,localhost:1111", "PILOSA_BIND": "localhost:1110", "PILOSA_TRANSLATION_MAP_SIZE": "100000"},
+			args: []string{"server",
+				"--anti-entropy.interval", "9m0s",
+				"--profile.block-rate", "4832",
+				"--profile.mutex-fraction", "8290",
+			},
+			env: map[string]string{
+				"PILOSA_CLUSTER_HOSTS":          "localhost:1110,localhost:1111",
+				"PILOSA_BIND":                   "localhost:1110",
+				"PILOSA_TRANSLATION_MAP_SIZE":   "100000",
+				"PILOSA_PROFILE_BLOCK_RATE":     "9123",
+				"PILOSA_PROFILE_MUTEX_FRACTION": "444",
+			},
 			cfgFileContent: `
 	bind = "localhost:0"
 	data-dir = "` + actualDataDir + `"
@@ -81,12 +102,17 @@ func TestServerConfig(t *testing.T) {
 		hosts = [
 			"localhost:19444",
 		]
+	[profile]
+		block-rate = 100
+		mutex-fraction = 10
 	`,
 			validation: func() error {
 				v := validator{}
 				v.Check(cmd.Server.Config.Cluster.Hosts, []string{"localhost:1110", "localhost:1111"})
 				v.Check(cmd.Server.Config.AntiEntropy.Interval, toml.Duration(time.Minute*9))
 				v.Check(cmd.Server.Config.Translation.MapSize, 100000)
+				v.Check(cmd.Server.Config.Profile.BlockRate, 4832)
+				v.Check(cmd.Server.Config.Profile.MutexFraction, 8290)
 				return v.Error()
 			},
 		},
@@ -106,6 +132,10 @@ func TestServerConfig(t *testing.T) {
 	[metric]
 		service = "statsd"
 		host = "127.0.0.1:8125"
+	[profile]
+		block-rate = 100
+		mutex-fraction = 10
+
 	`,
 			validation: func() error {
 				v := validator{}
@@ -114,6 +144,8 @@ func TestServerConfig(t *testing.T) {
 				v.Check(cmd.Server.Config.LogPath, logFile.Name())
 				v.Check(cmd.Server.Config.Metric.Service, "statsd")
 				v.Check(cmd.Server.Config.Metric.Host, "127.0.0.1:8125")
+				v.Check(cmd.Server.Config.Profile.BlockRate, 100)
+				v.Check(cmd.Server.Config.Profile.MutexFraction, 10)
 				if v.Error() != nil {
 					return v.Error()
 				}

ctl/server.go

@@ -79,4 +79,8 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 	flags.StringVarP(&srv.Config.Tracing.AgentHostPort, "tracing.agent-host-port", "", srv.Config.Tracing.AgentHostPort, "Jaeger agent host:port.")
 	flags.StringVarP(&srv.Config.Tracing.SamplerType, "tracing.sampler-type", "", srv.Config.Tracing.SamplerType, "Jaeger sampler type.")
 	flags.Float64VarP(&srv.Config.Tracing.SamplerParam, "tracing.sampler-param", "", srv.Config.Tracing.SamplerParam, "Jaeger sampler parameter.")
+
+	// Profiling
+	flags.IntVar(&srv.Config.Profile.BlockRate, "profile.block-rate", srv.Config.Profile.BlockRate, "Sampling rate for goroutine blocking profiler. One sample per <rate> ns.")
+	flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/<rate> of events.")
 }

docs/configuration.md

@@ -411,6 +411,32 @@ The config file is in the [toml format](https://github.com/toml-lang/toml) and h
     agent-host-port = "localhost:6831"
     ```
 
+#### Profile Block Rate
+
+* Description: Block Rate is passed directly to Go's
+  runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1
+  per `rate` nanoseconds. A value of "1" samples every event, and 0 disables
+  profiling.
+* Flag: `--profile.block-rate=10000000`
+* Env: `PILOSA_PROFILE_BLOCK_RATE=10000000`
+* Config:
+   ```toml
+   [profile]
+   block-rate = 10000000
+   ```
+
+#### Profile Mutex Fraction
+
+* Description: Mutex Fraction is passed directly to Go's
+  runtime.SetMutexProfileFraction. 1/`fraction` of events will be sampled. 
+* Flag: `--profile.mutex-fraction=100`
+* Env: `PILOSA_PROFILE_MUTEX_FRACTION=100`
+* Config:
+   ```toml
+   [profile]
+   mutex-fraction = 100
+   ```
+
 #### Translation Map Size
 
 * Description: Size in bytes of mmap to allocate for key translation

server/config.go

@@ -128,6 +128,13 @@ type Config struct {
 		// AgentHostPort is the host:port of the local agent.
 		AgentHostPort string `toml:"agent-host-port"`
 	} `toml:"tracing"`
+
+	Profile struct {
+		// BlockRate is passed directly to runtime.SetBlockProfileRate
+		BlockRate int `toml:"block-rate`
+		// MutexFraction is passed directly to runtime.SetMutexProfileFraction
+		MutexFraction int `toml:"mutex-fraction"`
+	} `toml:"profile"`
 }
 
 // NewConfig returns an instance of Config with default options.
@@ -170,6 +177,9 @@ func NewConfig() *Config {
 	c.Tracing.SamplerType = jaeger.SamplerTypeRemote
 	c.Tracing.SamplerParam = 0.001
 
+	c.Profile.BlockRate = 10000000 // 1 sample per 10 ms
+	c.Profile.MutexFraction = 100  // 1% sampling
+
 	return c
 }
 

server/server.go

@@ -29,6 +29,7 @@ import (
 	"net"
 	"os"
 	"os/signal"
+	"runtime"
 	"strconv"
 	"syscall"
 	"time"
@@ -179,6 +180,9 @@ func (m *Command) Wait() error {
 
 // SetupServer uses the cluster configuration to set up this server.
 func (m *Command) SetupServer() error {
+	runtime.SetBlockProfileRate(m.Config.Profile.BlockRate)
+	runtime.SetMutexProfileFraction(m.Config.Profile.MutexFraction)
+
 	syswrap.SetMaxMapCount(m.Config.MaxMapCount)
 	syswrap.SetMaxFileCount(m.Config.MaxFileCount)