From e469285fe38032015c0ccbdcb8da9d768bd496fb Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 18 Mar 2019 11:10:28 -0500 Subject: [PATCH] add fragment mmap tracking and limiting in the case that the map limit is reached, we'll fall back to reading the file into memory normally. --- ctl/server.go | 1 + fragment.go | 13 +++++++------ server/config.go | 6 ++++++ server/server.go | 3 +++ syswrap/mmap.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 syswrap/mmap.go diff --git a/ctl/server.go b/ctl/server.go index ae2c32d7f..10496c933 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -30,6 +30,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.IntVarP(&srv.Config.MaxWritesPerRequest, "max-writes-per-request", "", srv.Config.MaxWritesPerRequest, "Number of write commands per request.") flags.StringVar(&srv.Config.LogPath, "log-path", srv.Config.LogPath, "Log path") flags.BoolVar(&srv.Config.Verbose, "verbose", srv.Config.Verbose, "Enable verbose logging") + flags.Uint64Var(&srv.Config.MaxMapCount, "max-map-count", srv.Config.MaxMapCount, "Limits the maximum number of active mmaps. Pilosa will fall back to reading files once this is exhausted. Set below your system's vm.max_map_count.") // TLS SetTLSConfig(flags, &srv.Config.TLS.CertificatePath, &srv.Config.TLS.CertificateKeyPath, &srv.Config.TLS.SkipVerify) diff --git a/fragment.go b/fragment.go index 6675abf3a..81f1fb007 100644 --- a/fragment.go +++ b/fragment.go @@ -41,6 +41,7 @@ import ( "github.com/pilosa/pilosa/pql" "github.com/pilosa/pilosa/roaring" "github.com/pilosa/pilosa/stats" + "github.com/pilosa/pilosa/syswrap" "github.com/pilosa/pilosa/tracing" "github.com/pkg/errors" ) @@ -220,16 +221,16 @@ func (f *fragment) openStorage() error { } } else { // Mmap the underlying file so it can be zero copied. - data, err := syscall.Mmap(int(f.file.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) - if err != nil { - f.Logger.Printf("mmap failed %s using ReadAll", err) + data, err := syswrap.Mmap(int(f.file.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) + if err == syswrap.ErrMaxMapCountReached { + f.Logger.Debugf("maximum number of maps reached, reading file instead") data, err = ioutil.ReadAll(file) if err != nil { return errors.Wrap(err, "failure file readall") } - + } else if err != nil { + return errors.Wrap(err, "mmap failed") } else { - f.storageData = data // Advise the kernel that the mmap is accessed randomly. if err := madvise(f.storageData, syscall.MADV_RANDOM); err != nil { @@ -328,7 +329,7 @@ func (f *fragment) closeStorage() error { // Unmap the file. if f.storageData != nil { - if err := syscall.Munmap(f.storageData); err != nil { + if err := syswrap.Munmap(f.storageData); err != nil { return fmt.Errorf("munmap: %s", err) } f.storageData = nil diff --git a/server/config.go b/server/config.go index 9c787f92e..74e551024 100644 --- a/server/config.go +++ b/server/config.go @@ -70,6 +70,11 @@ type Config struct { AllowedOrigins []string `toml:"allowed-origins"` } `toml:"handler"` + // MaxMapCount puts an in-process limit on the number of mmaps. After this + // is exhausted, Pilosa will fall back to reading the file into memory + // normally. + MaxMapCount uint64 `toml:"max-map-count"` + // TLS TLS TLSConfig `toml:"tls"` @@ -124,6 +129,7 @@ func NewConfig() *Config { DataDir: "~/.pilosa", Bind: ":10101", MaxWritesPerRequest: 5000, + MaxMapCount: 60000, TLS: TLSConfig{}, } diff --git a/server/server.go b/server/server.go index 836b7fec7..f15965f52 100644 --- a/server/server.go +++ b/server/server.go @@ -45,6 +45,7 @@ import ( "github.com/pilosa/pilosa/logger" "github.com/pilosa/pilosa/stats" "github.com/pilosa/pilosa/statsd" + "github.com/pilosa/pilosa/syswrap" "github.com/pkg/errors" ) @@ -178,6 +179,8 @@ func (m *Command) Wait() error { // SetupServer uses the cluster configuration to set up this server. func (m *Command) SetupServer() error { + syswrap.MaxMapCount = m.Config.MaxMapCount + err := m.setupLogger() if err != nil { return errors.Wrap(err, "setting up logger") diff --git a/syswrap/mmap.go b/syswrap/mmap.go new file mode 100644 index 000000000..51186e55f --- /dev/null +++ b/syswrap/mmap.go @@ -0,0 +1,44 @@ +// Package syswrap wraps syscalls (just mmap right now) in order to impose a +// global in-process limit on the maximum number of active mmaps. +package syswrap + +import ( + "sync/atomic" + "syscall" + + "github.com/pkg/errors" +) + +var mapCount uint64 + +var ErrMaxMapCountReached = errors.New("maximum map count reached") + +// MaxMapCount default to slightly less than the typical +// default on Linux (65K). We want to leave some +// overhead for (e.g.) the Go runtime. +var MaxMapCount uint64 = 60000 + +// Mmap increments the global map count, and then calls syscall.Mmap. It +// decrements the map count and returns an error if the count was over the +// limit. If syscall.Mmap returns an error it also decrements the count. +func Mmap(fd int, offset int64, length int, prot int, flags int) (data []byte, err error) { + if newCount := atomic.AddUint64(&mapCount, 1); newCount > MaxMapCount { + atomic.AddUint64(&mapCount, ^uint64(0)) // decrement + return nil, ErrMaxMapCountReached + } + data, err = syscall.Mmap(fd, offset, length, prot, flags) + if err != nil { + atomic.AddUint64(&mapCount, ^uint64(0)) // decrement + } + return data, err +} + +// Munmap calls sycall.Munmap, and then decrements the global map count if there +// was no error. +func Munmap(b []byte) (err error) { + err = syscall.Munmap(b) + if err == nil { + atomic.AddUint64(&mapCount, ^uint64(0)) // decrement + } + return err +}