scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 09:05:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Attempt to set open file limit during Pilosa startup. Fixes #722.

Browse source
This commit is contained in:
Cody Soyland 2017-07-25 16:51:03 -05:00
parent 3c9ca6c2c4
commit 0df72e3d33
1 changed files with 46 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
holder.go
View file

@ -24,11 +24,17 @@ import (
"path/filepath"
"sort"
"sync"
"syscall"
"time"
)
// DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
const DefaultCacheFlushInterval = 1 * time.Minute
const (
// DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
DefaultCacheFlushInterval = 1 * time.Minute
// FileLimit is the maximum open file limit (ulimit -n) to automatically set.
FileLimit uint64 = 262144 // (512^2)
)
// Holder represents a container for indexes.
type Holder struct {
@ -71,6 +77,8 @@ func NewHolder() *Holder {
// Open initializes the root data directory for the holder.
func (h *Holder) Open() error {
h.setFileLimit()
if err := os.MkdirAll(h.Path, 0777); err != nil {
return err
}
@ -351,6 +359,42 @@ func (h *Holder) flushCaches() {
}
}
// setFileLimit attempts to set the open file limit to the FileLimit constant defined above.
func (h *Holder) setFileLimit() {
lim := &syscall.Rlimit{}
var setLimit uint64
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, lim); err != nil {
h.logger().Printf("ERROR checking open file limit: %s", err)
return
}
if lim.Max < FileLimit {
h.logger().Printf("WARNING: open file limit max is %d. Please consider running \"ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error.", lim.Max, FileLimit)
}
// If the soft limit is lower than both the FileLimit constant and the hard limit, we will try to change it.
if lim.Cur < FileLimit && lim.Cur < lim.Max {
if FileLimit < lim.Max {
setLimit = FileLimit
} else {
setLimit = lim.Max
}
lim.Cur = setLimit
// Try to set the limit
if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, lim); err != nil {
h.logger().Printf("ERROR setting open file limit: %s", err)
}
// Check the limit after setting it. OS may not obey Setrlimit call.
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, lim); err != nil {
h.logger().Printf("ERROR checking open file limit: %s", err)
} else {
if lim.Cur != setLimit {
h.logger().Printf("WARNING: Tried to set open file limit to %d, but it is %d. You may consider running \"ulimit -n %d\" before starting Pilosa to avoid \"too many open files\" error.", setLimit, lim.Cur, FileLimit)
}
}
}
}
func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) }
// HolderSyncer is an active anti-entropy tool that compares the local holder