diff --git a/ctl/server.go b/ctl/server.go index 90fdfb46e..194633a48 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -45,6 +45,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // Translation flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.") + flags.IntVarP(&srv.Config.Translation.MapSize, "translation.map-size", "", srv.Config.Translation.MapSize, "Size of mmap to allocate for key translation.") // Gossip flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.") diff --git a/holder.go b/holder.go index 794104e53..5cffeea9b 100644 --- a/holder.go +++ b/holder.go @@ -77,9 +77,19 @@ type Holder struct { Logger Logger } +// HolderOption is a functional option type for pilosa.Holder +type HolderOption func(f *Holder) error + +func OptHolderTranslateFileMapSize(mapSize int) HolderOption { + return func(h *Holder) error { + h.translateFile = NewTranslateFile(OptTranslateFileMapSize(mapSize)) + return nil + } +} + // NewHolder returns a new instance of Holder. -func NewHolder() *Holder { - return &Holder{ +func NewHolder(opts ...HolderOption) *Holder { + h := &Holder{ indexes: make(map[string]*Index), closing: make(chan struct{}), @@ -97,6 +107,15 @@ func NewHolder() *Holder { Logger: NopLogger, } + + for _, opt := range opts { + err := opt(h) + if err != nil { + // TODO (2.0): Change func signature to return error + panic(errors.Wrap(err, "applying option")) + } + } + return h } // Open initializes the root data directory for the holder. diff --git a/server.go b/server.go index cde1dd406..708fd853b 100644 --- a/server.go +++ b/server.go @@ -234,6 +234,18 @@ func OptServerClusterHasher(h Hasher) ServerOption { } } +func OptServerHolderOptions(opts ...HolderOption) ServerOption { + return func(s *Server) error { + for _, opt := range opts { + err := opt(s.holder) + if err != nil { + return errors.Wrap(err, "applying option") + } + } + return nil + } +} + // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { s := &Server{ diff --git a/server/config.go b/server/config.go index 8367eb835..d255e4bb6 100644 --- a/server/config.go +++ b/server/config.go @@ -71,8 +71,9 @@ type Config struct { // Gossip config is based around memberlist.Config. Gossip gossip.Config `toml:"gossip"` - // DEPRECATED: Translation config supports translation store replication. Translation struct { + MapSize int `toml:"map-size"` + // DEPRECATED: Translation config supports translation store replication. PrimaryURL string `toml:"primary-url"` } `toml:"translation"` diff --git a/server/server.go b/server/server.go index 4fdd711fe..61942522f 100644 --- a/server/server.go +++ b/server/server.go @@ -283,6 +283,16 @@ func (m *Command) SetupServer() error { coordinatorOpt, } + if m.Config.Translation.MapSize > 0 { + serverOptions = append( + serverOptions, + pilosa.OptServerHolderOptions( + pilosa.OptHolderTranslateFileMapSize( + m.Config.Translation.MapSize, + ), + ), + ) + } serverOptions = append(serverOptions, m.serverOptions...) m.Server, err = pilosa.NewServer(serverOptions...) diff --git a/test/pilosa.go b/test/pilosa.go index 9dbd78057..835291cbd 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -55,11 +55,22 @@ func newCommand(opts ...server.CommandOption) *Command { panic(err) } - // set aggressive close timeout by default to avoid hanging tests. This was + // Set aggressive close timeout by default to avoid hanging tests. This was // a problem with PDK tests which used go-pilosa as well. We put it at the // beginning of the option slice so that it can be overridden by user-passed // options. - opts = append([]server.CommandOption{server.OptCommandCloseTimeout(time.Millisecond * 2)}, opts...) + // Also set TranslateFile MapSize to a smaller number so memory allocation + // does not fail on 32-bit systems. + opts = append([]server.CommandOption{ + server.OptCommandCloseTimeout(time.Millisecond * 2), + server.OptCommandServerOptions( + pilosa.OptServerHolderOptions( + pilosa.OptHolderTranslateFileMapSize( + 2 << 26, + ), + ), + ), + }, opts...) m := &Command{commandOptions: opts} m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...) m.Config.DataDir = path diff --git a/translate.go b/translate.go index 41f3b4852..f1eddfbfe 100644 --- a/translate.go +++ b/translate.go @@ -5,7 +5,6 @@ import ( "bytes" "context" "encoding/binary" - "errors" "fmt" "io" "io/ioutil" @@ -17,6 +16,7 @@ import ( "time" "github.com/cespare/xxhash" + "github.com/pkg/errors" ) const ( @@ -81,9 +81,26 @@ type TranslateFile struct { replicationRetryInterval time.Duration } +// TranslateFileOption is a functional option type for pilosa.TranslateFile +type TranslateFileOption func(f *TranslateFile) error + +func OptTranslateFileMapSize(mapSize int) TranslateFileOption { + return func(f *TranslateFile) error { + f.mapSize = mapSize + return nil + } +} + // NewTranslateFile returns a new instance of TranslateFile. -func NewTranslateFile() *TranslateFile { - return &TranslateFile{ +func NewTranslateFile(opts ...TranslateFileOption) *TranslateFile { + // 10GB default map size + defaultMapSize := 10 * (1 << 30) + // Use 2GB default map size on 32-bit systems + if 32<<(^uint(0)>>32&1) == 32 { + defaultMapSize = (1 << 31) - 1 // 2GB + } + + f := &TranslateFile{ writeNotify: make(chan struct{}), closing: make(chan struct{}), cols: make(map[string]*index), @@ -96,6 +113,16 @@ func NewTranslateFile() *TranslateFile { replicationRetryInterval: defaultReplicationRetryInterval, } + + for _, opt := range opts { + err := opt(f) + if err != nil { + // TODO (2.0): Change func signature to return error + panic(errors.Wrap(err, "applying option")) + } + } + + return f } func (s *TranslateFile) Open() (err error) { diff --git a/translate_mapsize_386.go b/translate_mapsize_386.go deleted file mode 100644 index 259b735ac..000000000 --- a/translate_mapsize_386.go +++ /dev/null @@ -1,5 +0,0 @@ -package pilosa - -// defaultMapSize is the default size of mapped memory for the translate store. -// It is passed as an int to syscall.Mmap and so must be < 2^31 -const defaultMapSize = (1 << 31) - 1 // 2GB diff --git a/translate_mapsize_all64bitsystems.go b/translate_mapsize_all64bitsystems.go deleted file mode 100644 index 605d6270a..000000000 --- a/translate_mapsize_all64bitsystems.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build !386 - -package pilosa - -// defaultMapSize is the default size of mapped memory for the translate store. -// It is passed as an int to syscall.Mmap and so can only be larger than 2^31 on -// 64bit systems. -const defaultMapSize = 10 * (1 << 30) // 10GB