Merge pull request #636 from raskle/429-stats-file-descriptor

Count open file handles as a StatsD metric.

docs/administration.md

@@ -157,3 +157,5 @@ We currently track the following events
 <strong id="garbage_collection">Garbage Collection:</strong> Event count when Garbage Collection occurs.
 
 <strong id="goroutines">Goroutines:</strong> Number of running Goroutines.
+
+<strong id="openfiles">OpenFiles:</strong> Number of open file handles associated with running Pilosa process ID.

server.go

@@ -24,8 +24,10 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"os/exec"
 	"runtime"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -486,6 +488,9 @@ func (s *Server) monitorRuntime() {
 		// Record the number of go routines
 		s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()), 1.0)
 
+		// Open File handles
+		s.Holder.Stats.Gauge("OpenFiles", float64(CountOpenFiles()), 1.0)
+
 		// Runtime memory metrics
 		runtime.ReadMemStats(&m)
 		s.Holder.Stats.Gauge("HeapAlloc", float64(m.HeapAlloc), 1.0)
@@ -496,6 +501,29 @@ func (s *Server) monitorRuntime() {
 	}
 }
 
+// CountOpenFiles on opperating systems that support lsof
+func CountOpenFiles() int {
+	count := 0
+
+	switch runtime.GOOS {
+	case "darwin", "linux", "unix", "freebsd":
+		// -b option avoid kernel blocks
+		pid := os.Getpid()
+		out, err := exec.Command("/bin/sh", "-c", fmt.Sprintf("lsof -b -p %v", pid)).Output()
+		if err != nil {
+			log.Fatal(err)
+		}
+		// only count lines with our pid, avoiding warning messages from -b
+		lines := strings.Split(string(out), strconv.Itoa(pid))
+		count = len(lines)
+	case "windows":
+		// TODO: count open file handles on windows
+	default:
+
+	}
+	return count
+}
+
 // StatusHandler specifies two methods which an object must implement to share
 // state in the cluster. These are used by the GossipNodeSet to implement the
 // LocalState and MergeRemoteState methods of memberlist.Delegate

server/server_test.go

@@ -25,7 +25,9 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"reflect"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -372,6 +374,42 @@ path = "/path/to/plugins"
 	}
 }
 
+// tempMkdir makes a temporary directory
+func tempMkdir(t *testing.T) string {
+	dir, err := ioutil.TempDir("", "pilosatemp")
+	if err != nil {
+		t.Fatalf("failed to create test directory: %s", err)
+	}
+	return dir
+}
+
+// Ensure the file handle count is working
+func TestCountOpenFiles(t *testing.T) {
+	// Windows is not supported yet
+	supported := []string{"darwin", "linux", "unix", "freebsd"}
+	sort.Strings(supported)
+	i := sort.Search(len(supported),
+		func(i int) bool { return supported[i] >= runtime.GOOS })
+	if i == len(supported) {
+		return
+	}
+
+	// Create directory store temp file
+	testDir := tempMkdir(t)
+	defer os.RemoveAll(testDir)
+
+	count := pilosa.CountOpenFiles()
+	testFile := filepath.Join(testDir, "test.txt")
+	_, err := os.Create(testFile)
+	if err != nil {
+		t.Fatalf("create test file failed: %s", err)
+	}
+
+	if pilosa.CountOpenFiles() < count+1 {
+		t.Error("Invalid open file handle count")
+	}
+}
+
 // Ensure program can send/receive broadcast messages.
 func TestMain_SendReceiveMessage(t *testing.T) {
 	m0 := MustRunMain()