Merge pull request #748 from codysoyland/722-ulimit

Attempt to set open file limit during Pilosa startup. Fixes #722.

docs/administration.md

@@ -30,6 +30,12 @@ Pilosa is designed to be a distributed application, with data replication shared
 
 While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available.  You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced.   Nor is it advisable to have many small machines.  The internode network traffic will become a bottleneck.  You can always add nodes later, but that does require some down time.
 
+### Open File Limits
+
+Pilosa requires a large number of open files to support its memory-mapped file storage system. Most operating systems put limits on the maximum number of files that may be opened concurrently by a process. On Linux systems, this limit is controlled by a utility called [ulimit](https://ss64.com/bash/ulimit.html). Pilosa will automatically attempt to raise the limit to `262144` during startup, but it may fail due to access limitations. If you see errors related to open file limits when starting Pilosa, it is recommended that you run `sudo ulimit -n 262144` before starting Pilosa.
+
+On Mac OS X, `ulimit` does not behave predictably. [This blog post](https://blog.dekstroza.io/ulimit-shenanigans-on-osx-el-capitan/) contains information about setting open file limits in OS X.
+
 ### Importing and Exporting Data
 
 #### Importing

docs/getting-started.md

@@ -8,7 +8,7 @@ Pilosa supports an HTTP interface which uses JSON by default.
 Any HTTP tool can be used to interact with the Pilosa server. The examples in this documentation will use [curl](https://curl.haxx.se/) which is available by default on many UNIX-like systems including Linux and MacOS. Windows users can download curl [here](https://curl.haxx.se/download.html).
 
 <div class="note">
-    <p>Note that Pilosa server requires a high limit for open files. Check the documentation of your system to see how to increase it in case you hit that limit.</p>
+    <p>Note that Pilosa server requires a high limit for open files. Check the documentation of your system to see how to increase it in case you hit that limit. See <a href="/docs/administration/#open-file-limits">Open File Limits</a> for more details.</p>
 </div>
 
 ### Starting Pilosa

holder.go

@@ -24,11 +24,17 @@ import (
 	"path/filepath"
 	"sort"
 	"sync"
+	"syscall"
 	"time"
 )
 
-// DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
-const DefaultCacheFlushInterval = 1 * time.Minute
+const (
+	// DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
+	DefaultCacheFlushInterval = 1 * time.Minute
+
+	// FileLimit is the maximum open file limit (ulimit -n) to automatically set.
+	FileLimit = 262144 // (512^2)
+)
 
 // Holder represents a container for indexes.
 type Holder struct {
@@ -71,6 +77,8 @@ func NewHolder() *Holder {
 
 // Open initializes the root data directory for the holder.
 func (h *Holder) Open() error {
+	h.setFileLimit()
+
 	if err := os.MkdirAll(h.Path, 0777); err != nil {
 		return err
 	}
@@ -351,6 +359,55 @@ func (h *Holder) flushCaches() {
 	}
 }
 
+// setFileLimit attempts to set the open file limit to the FileLimit constant defined above.
+func (h *Holder) setFileLimit() {
+	oldLimit := &syscall.Rlimit{}
+	newLimit := &syscall.Rlimit{}
+
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil {
+		h.logger().Printf("ERROR checking open file limit: %s", err)
+		return
+	}
+	// If the soft limit is lower than the FileLimit constant, we will try to change it.
+	if oldLimit.Cur < FileLimit {
+		newLimit.Cur = FileLimit
+		// If the hard limit is not high enough, we will try to change it too.
+		if oldLimit.Max < FileLimit {
+			newLimit.Max = FileLimit
+		} else {
+			newLimit.Max = oldLimit.Max
+		}
+
+		// Try to set the limit
+		if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, newLimit); err != nil {
+			// If we just tried to change the hard limit and failed, we probably don't have permission. Let's try again without setting the hard limit.
+			if newLimit.Max > oldLimit.Max {
+				newLimit.Max = oldLimit.Max
+				// Obviously the hard limit cannot be higher than the soft limit.
+				if newLimit.Cur >= newLimit.Max {
+					newLimit.Cur = newLimit.Max
+				}
+				// Try setting again with lowered Max (hard limit)
+				if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, newLimit); err != nil {
+					h.logger().Printf("ERROR setting open file limit: %s", err)
+				}
+				// If we weren't trying to change the hard limit, let the user know something is wrong.
+			} else {
+				h.logger().Printf("ERROR setting open file limit: %s", err)
+			}
+		}
+
+		// Check the limit after setting it. OS may not obey Setrlimit call.
+		if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil {
+			h.logger().Printf("ERROR checking open file limit: %s", err)
+		} else {
+			if oldLimit.Cur < FileLimit {
+				h.logger().Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit)
+			}
+		}
+	}
+}
+
 func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) }
 
 // HolderSyncer is an active anti-entropy tool that compares the local holder