measure runtime stats

added package to measure when garbage collection occurs

glide.lock

@@ -1,5 +1,5 @@
-hash: 4bdea17c62dcd469584382515052e7a246fae6deefc2d62da70bf768e18e1f0c
-updated: 2017-04-19T10:51:10.409094081-05:00
+hash: a6889bf5334164aa4aebd670eee45444f1bdfa1888642d290acf33e18e57f293
+updated: 2017-04-19T17:19:43.584486333-05:00
 imports:
 - name: github.com/armon/go-metrics
   version: 97c69685293dce4c0a2d0b19535179bbc976e4d2
@@ -7,6 +7,8 @@ imports:
   version: 4b1ebc1869ad66568b313d0dc410e2be72670dda
 - name: github.com/BurntSushi/toml
   version: 99064174e013895bbd9b025c31100bd1d9b590ca
+- name: github.com/CAFxX/gcnotifier
+  version: adea3e70515666981da25214a7d3e377e4841c22
 - name: github.com/DataDog/datadog-go
   version: 909c02b65dd8a52e8fa6072db9752a112227cf21
   subpackages:
@@ -46,10 +48,10 @@ imports:
   subpackages:
   - hcl/ast
   - hcl/parser
-  - hcl/scanner
-  - hcl/strconv
   - hcl/token
   - json/parser
+  - hcl/scanner
+  - hcl/strconv
   - json/scanner
   - json/token
 - name: github.com/hashicorp/memberlist
@@ -101,4 +103,4 @@ imports:
   - unicode/norm
 - name: gopkg.in/yaml.v2
   version: a3f3340b5840cee44f372bddb5880fcbc419b46a
-testImports: []
+devImports: []

glide.yaml

@@ -33,3 +33,4 @@ import:
   version: ^1.3.0
 - package: github.com/hashicorp/memberlist
 - package: golang.org/x/sync
+- package: github.com/CAFxX/gcnotifier

server.go

@@ -10,10 +10,12 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"runtime"
 	"strconv"
 	"sync"
 	"time"
 
+	"github.com/CAFxX/gcnotifier"
 	"github.com/gogo/protobuf/proto"
 	"github.com/pilosa/pilosa/internal"
 )
@@ -46,6 +48,7 @@ type Server struct {
 	// Background monitoring intervals.
 	AntiEntropyInterval time.Duration
 	PollingInterval     time.Duration
+	MetricInterval      time.Duration
 
 	LogOutput io.Writer
 }
@@ -62,6 +65,7 @@ func NewServer() *Server {
 
 		AntiEntropyInterval: DefaultAntiEntropyInterval,
 		PollingInterval:     DefaultPollingInterval,
+		MetricInterval:      0,
 
 		LogOutput: os.Stderr,
 	}
@@ -131,9 +135,10 @@ func (s *Server) Open() error {
 	go func() { http.Serve(ln, s.Handler) }()
 
 	// Start background monitoring.
-	s.wg.Add(2)
+	s.wg.Add(3)
 	go func() { defer s.wg.Done(); s.monitorAntiEntropy() }()
 	go func() { defer s.wg.Done(); s.monitorMaxSlices() }()
+	go func() { defer s.wg.Done(); s.monitorRuntime() }()
 
 	return nil
 }
@@ -363,3 +368,37 @@ func checkMaxSlices(hostport string) (map[string]uint64, error) {
 
 	return pb.MaxSlices, nil
 }
+
+// monitorRuntime periodically polls the Go runtime metrics.
+func (s *Server) monitorRuntime() {
+	if s.MetricInterval > 0 {
+		ticker := time.NewTicker(s.MetricInterval)
+		defer ticker.Stop()
+
+		gcn := gcnotifier.New()
+		defer gcn.Close()
+
+		s.logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval)
+
+		for {
+			// Wait for tick or a close.
+			select {
+			case <-s.closing:
+				return
+			case <-gcn.AfterGC():
+				// GC just ran
+				s.Index.Stats.Count("garbage_collection", 1)
+				s.logger().Printf("garbage collection complete")
+			case <-ticker.C:
+			}
+
+			s.logger().Printf("runtime stats  beginning")
+
+			// TODO
+			s.Index.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()))
+
+			// Record successful sync in log.
+			s.logger().Printf("runtime stats  complete")
+		}
+	}
+}