From 0df72e3d33588f2d0c0e58d60395a0fb409dc36a Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 25 Jul 2017 16:51:03 -0500 Subject: [PATCH 1/4] Attempt to set open file limit during Pilosa startup. Fixes #722. --- holder.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/holder.go b/holder.go index 59fef1560..31ae20b3d 100644 --- a/holder.go +++ b/holder.go @@ -24,11 +24,17 @@ import ( "path/filepath" "sort" "sync" + "syscall" "time" ) -// DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. -const DefaultCacheFlushInterval = 1 * time.Minute +const ( + // DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval. + DefaultCacheFlushInterval = 1 * time.Minute + + // FileLimit is the maximum open file limit (ulimit -n) to automatically set. + FileLimit uint64 = 262144 // (512^2) +) // Holder represents a container for indexes. type Holder struct { @@ -71,6 +77,8 @@ func NewHolder() *Holder { // Open initializes the root data directory for the holder. func (h *Holder) Open() error { + h.setFileLimit() + if err := os.MkdirAll(h.Path, 0777); err != nil { return err } @@ -351,6 +359,42 @@ func (h *Holder) flushCaches() { } } +// setFileLimit attempts to set the open file limit to the FileLimit constant defined above. +func (h *Holder) setFileLimit() { + lim := &syscall.Rlimit{} + var setLimit uint64 + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, lim); err != nil { + h.logger().Printf("ERROR checking open file limit: %s", err) + return + } + if lim.Max < FileLimit { + h.logger().Printf("WARNING: open file limit max is %d. Please consider running \"ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error.", lim.Max, FileLimit) + } + // If the soft limit is lower than both the FileLimit constant and the hard limit, we will try to change it. + if lim.Cur < FileLimit && lim.Cur < lim.Max { + if FileLimit < lim.Max { + setLimit = FileLimit + } else { + setLimit = lim.Max + } + lim.Cur = setLimit + + // Try to set the limit + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, lim); err != nil { + h.logger().Printf("ERROR setting open file limit: %s", err) + } + + // Check the limit after setting it. OS may not obey Setrlimit call. + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, lim); err != nil { + h.logger().Printf("ERROR checking open file limit: %s", err) + } else { + if lim.Cur != setLimit { + h.logger().Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error.", setLimit, lim.Cur, FileLimit) + } + } + } +} + func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) } // HolderSyncer is an active anti-entropy tool that compares the local holder From ed6e063102894ed19e7538fe539353415499206d Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 26 Jul 2017 17:08:51 -0500 Subject: [PATCH 2/4] Attempt to set hard open file limit, improve error messages, and add documentation about open file limits. --- docs/administration.md | 6 ++++++ docs/getting-started.md | 2 +- holder.go | 47 ++++++++++++++++++++++++++--------------- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index e562f94ce..d1f6aefa5 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -30,6 +30,12 @@ Pilosa is designed to be a distributed application, with data replication shared While Pilosa does have some high system requirements it is not a best practice to set up a cluster with the fewest, largest machines available. You want an evenly distributed load across several nodes in a cluster to easily recover from a single node failure, and have the resource capacity to handle a missing node until it's repaired or replaced. Nor is it advisable to have many small machines. The internode network traffic will become a bottleneck. You can always add nodes later, but that does require some down time. +### Open File Limits + +Pilosa requires a large amount of open files to support its memory-mapped file storage system. Most operating systems put limits on the maximum number of files that may be opened concurrently by a process. On Linux systems, this limit is controlled by a utility called [ulimit](https://ss64.com/bash/ulimit.html). Pilosa will automatically attempt to raise the limit to `262144` during startup, but it may fail due to access limitations. If you see errors related to open file limits when starting Pilosa, it is recommended that you run `sudo ulimit -n 262144` before starting Pilosa. + +On Mac OS X, `ulimit` does not behave predictably. [This blog post](https://blog.dekstroza.io/ulimit-shenanigans-on-osx-el-capitan/) contains information about setting open file limits in OS X. + ### Importing and Exporting Data #### Importing diff --git a/docs/getting-started.md b/docs/getting-started.md index 96fd4e9e7..eb30f064b 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -8,7 +8,7 @@ Pilosa supports an HTTP interface which uses JSON by default. Any HTTP tool can be used to interact with the Pilosa server. The examples in this documentation will use [curl](https://curl.haxx.se/) which is available by default on many UNIX-like systems including Linux and MacOS. Windows users can download curl [here](https://curl.haxx.se/download.html).
-

Note that Pilosa server requires a high limit for open files. Check the documentation of your system to see how to increase it in case you hit that limit.

+

Note that Pilosa server requires a high limit for open files. Check the documentation of your system to see how to increase it in case you hit that limit. See Open File Limits for more details.

### Starting Pilosa diff --git a/holder.go b/holder.go index 31ae20b3d..8de3419ac 100644 --- a/holder.go +++ b/holder.go @@ -361,35 +361,48 @@ func (h *Holder) flushCaches() { // setFileLimit attempts to set the open file limit to the FileLimit constant defined above. func (h *Holder) setFileLimit() { - lim := &syscall.Rlimit{} - var setLimit uint64 - if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, lim); err != nil { + oldLimit := &syscall.Rlimit{} + newLimit := &syscall.Rlimit{} + + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil { h.logger().Printf("ERROR checking open file limit: %s", err) return } - if lim.Max < FileLimit { - h.logger().Printf("WARNING: open file limit max is %d. Please consider running \"ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error.", lim.Max, FileLimit) - } - // If the soft limit is lower than both the FileLimit constant and the hard limit, we will try to change it. - if lim.Cur < FileLimit && lim.Cur < lim.Max { - if FileLimit < lim.Max { - setLimit = FileLimit + // If the soft limit is lower than the FileLimit constant, we will try to change it. + if oldLimit.Cur < FileLimit { + newLimit.Cur = FileLimit + // If the hard limit is not high enough, we will try to change it too. + if oldLimit.Max < FileLimit { + newLimit.Max = FileLimit } else { - setLimit = lim.Max + newLimit.Max = oldLimit.Max } - lim.Cur = setLimit // Try to set the limit - if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, lim); err != nil { - h.logger().Printf("ERROR setting open file limit: %s", err) + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, newLimit); err != nil { + // If we just tried to change the hard limit and failed, we probably don't have permission. Let's try again without setting the hard limit. + if newLimit.Max > oldLimit.Max { + newLimit.Max = oldLimit.Max + // Obviously the hard limit cannot be higher than the soft limit. + if newLimit.Cur >= newLimit.Max { + newLimit.Cur = newLimit.Max + } + // Try setting again with lowered Max (hard limit) + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, newLimit); err != nil { + h.logger().Printf("ERROR setting open file limit: %s", err) + } + // If we weren't trying to change the hard limit, let the user know something is wrong. + } else { + h.logger().Printf("ERROR setting open file limit: %s", err) + } } // Check the limit after setting it. OS may not obey Setrlimit call. - if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, lim); err != nil { + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, oldLimit); err != nil { h.logger().Printf("ERROR checking open file limit: %s", err) } else { - if lim.Cur != setLimit { - h.logger().Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error.", setLimit, lim.Cur, FileLimit) + if oldLimit.Cur < FileLimit { + h.logger().Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"sudo ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error. See https://www.pilosa.com/docs/administration/#open-file-limits for more information.", FileLimit, oldLimit.Cur, FileLimit) } } } From f99a15bed239e080ea72d1070bdde74f0677d27b Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 27 Jul 2017 13:58:26 -0500 Subject: [PATCH 3/4] Change "amount" to "number" in docs. --- docs/administration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration.md b/docs/administration.md index d1f6aefa5..04b1b2bf0 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -32,7 +32,7 @@ While Pilosa does have some high system requirements it is not a best practice t ### Open File Limits -Pilosa requires a large amount of open files to support its memory-mapped file storage system. Most operating systems put limits on the maximum number of files that may be opened concurrently by a process. On Linux systems, this limit is controlled by a utility called [ulimit](https://ss64.com/bash/ulimit.html). Pilosa will automatically attempt to raise the limit to `262144` during startup, but it may fail due to access limitations. If you see errors related to open file limits when starting Pilosa, it is recommended that you run `sudo ulimit -n 262144` before starting Pilosa. +Pilosa requires a large number of open files to support its memory-mapped file storage system. Most operating systems put limits on the maximum number of files that may be opened concurrently by a process. On Linux systems, this limit is controlled by a utility called [ulimit](https://ss64.com/bash/ulimit.html). Pilosa will automatically attempt to raise the limit to `262144` during startup, but it may fail due to access limitations. If you see errors related to open file limits when starting Pilosa, it is recommended that you run `sudo ulimit -n 262144` before starting Pilosa. On Mac OS X, `ulimit` does not behave predictably. [This blog post](https://blog.dekstroza.io/ulimit-shenanigans-on-osx-el-capitan/) contains information about setting open file limits in OS X. From bd4e8285172e1d33deac661405770d9e6c9b6322 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 27 Jul 2017 14:19:19 -0500 Subject: [PATCH 4/4] Use untyped constant to allow building on FreeBSD/Dragonfly --- holder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/holder.go b/holder.go index 8de3419ac..f7c363524 100644 --- a/holder.go +++ b/holder.go @@ -33,7 +33,7 @@ const ( DefaultCacheFlushInterval = 1 * time.Minute // FileLimit is the maximum open file limit (ulimit -n) to automatically set. - FileLimit uint64 = 262144 // (512^2) + FileLimit = 262144 // (512^2) ) // Holder represents a container for indexes.