diff --git a/glide.lock b/glide.lock index c142737ff..7ba5cdab6 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 4bdea17c62dcd469584382515052e7a246fae6deefc2d62da70bf768e18e1f0c -updated: 2017-04-19T10:51:10.409094081-05:00 +hash: a6889bf5334164aa4aebd670eee45444f1bdfa1888642d290acf33e18e57f293 +updated: 2017-04-19T17:19:43.584486333-05:00 imports: - name: github.com/armon/go-metrics version: 97c69685293dce4c0a2d0b19535179bbc976e4d2 @@ -7,6 +7,8 @@ imports: version: 4b1ebc1869ad66568b313d0dc410e2be72670dda - name: github.com/BurntSushi/toml version: 99064174e013895bbd9b025c31100bd1d9b590ca +- name: github.com/CAFxX/gcnotifier + version: adea3e70515666981da25214a7d3e377e4841c22 - name: github.com/DataDog/datadog-go version: 909c02b65dd8a52e8fa6072db9752a112227cf21 subpackages: @@ -46,10 +48,10 @@ imports: subpackages: - hcl/ast - hcl/parser - - hcl/scanner - - hcl/strconv - hcl/token - json/parser + - hcl/scanner + - hcl/strconv - json/scanner - json/token - name: github.com/hashicorp/memberlist @@ -101,4 +103,4 @@ imports: - unicode/norm - name: gopkg.in/yaml.v2 version: a3f3340b5840cee44f372bddb5880fcbc419b46a -testImports: [] +devImports: [] diff --git a/glide.yaml b/glide.yaml index a7c0e98eb..4b1ff6038 100644 --- a/glide.yaml +++ b/glide.yaml @@ -33,3 +33,4 @@ import: version: ^1.3.0 - package: github.com/hashicorp/memberlist - package: golang.org/x/sync +- package: github.com/CAFxX/gcnotifier diff --git a/server.go b/server.go index 229214805..68337049d 100644 --- a/server.go +++ b/server.go @@ -10,10 +10,12 @@ import ( "net/http" "net/url" "os" + "runtime" "strconv" "sync" "time" + "github.com/CAFxX/gcnotifier" "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) @@ -46,6 +48,7 @@ type Server struct { // Background monitoring intervals. AntiEntropyInterval time.Duration PollingInterval time.Duration + MetricInterval time.Duration LogOutput io.Writer } @@ -62,6 +65,7 @@ func NewServer() *Server { AntiEntropyInterval: DefaultAntiEntropyInterval, PollingInterval: DefaultPollingInterval, + MetricInterval: 0, LogOutput: os.Stderr, } @@ -131,9 +135,10 @@ func (s *Server) Open() error { go func() { http.Serve(ln, s.Handler) }() // Start background monitoring. - s.wg.Add(2) + s.wg.Add(3) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() go func() { defer s.wg.Done(); s.monitorMaxSlices() }() + go func() { defer s.wg.Done(); s.monitorRuntime() }() return nil } @@ -363,3 +368,37 @@ func checkMaxSlices(hostport string) (map[string]uint64, error) { return pb.MaxSlices, nil } + +// monitorRuntime periodically polls the Go runtime metrics. +func (s *Server) monitorRuntime() { + if s.MetricInterval > 0 { + ticker := time.NewTicker(s.MetricInterval) + defer ticker.Stop() + + gcn := gcnotifier.New() + defer gcn.Close() + + s.logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval) + + for { + // Wait for tick or a close. + select { + case <-s.closing: + return + case <-gcn.AfterGC(): + // GC just ran + s.Index.Stats.Count("garbage_collection", 1) + s.logger().Printf("garbage collection complete") + case <-ticker.C: + } + + s.logger().Printf("runtime stats beginning") + + // TODO + s.Index.Stats.Gauge("goroutines", float64(runtime.NumGoroutine())) + + // Record successful sync in log. + s.logger().Printf("runtime stats complete") + } + } +}