Make translate map size configurable.

ctl/server.go

@@ -45,6 +45,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
 
 	// Translation
 	flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.")
+	flags.IntVarP(&srv.Config.Translation.MapSize, "translation.map-size", "", srv.Config.Translation.MapSize, "Size of mmap to allocate for key translation.")
 
 	// Gossip
 	flags.StringVarP(&srv.Config.Gossip.Port, "gossip.port", "", srv.Config.Gossip.Port, "Port to which pilosa should bind for internal state sharing.")

holder.go

@@ -77,9 +77,19 @@ type Holder struct {
 	Logger Logger
 }
 
+// HolderOption is a functional option type for pilosa.Holder
+type HolderOption func(f *Holder) error
+
+func OptHolderTranslateFileMapSize(mapSize int) HolderOption {
+	return func(h *Holder) error {
+		h.translateFile = NewTranslateFile(OptTranslateFileMapSize(mapSize))
+		return nil
+	}
+}
+
 // NewHolder returns a new instance of Holder.
-func NewHolder() *Holder {
-	return &Holder{
+func NewHolder(opts ...HolderOption) *Holder {
+	h := &Holder{
 		indexes: make(map[string]*Index),
 		closing: make(chan struct{}),
 
@@ -97,6 +107,15 @@ func NewHolder() *Holder {
 
 		Logger: NopLogger,
 	}
+
+	for _, opt := range opts {
+		err := opt(h)
+		if err != nil {
+			// TODO (2.0): Change func signature to return error
+			panic(errors.Wrap(err, "applying option"))
+		}
+	}
+	return h
 }
 
 // Open initializes the root data directory for the holder.

server.go

@@ -234,6 +234,18 @@ func OptServerClusterHasher(h Hasher) ServerOption {
 	}
 }
 
+func OptServerHolderOptions(opts ...HolderOption) ServerOption {
+	return func(s *Server) error {
+		for _, opt := range opts {
+			err := opt(s.holder)
+			if err != nil {
+				return errors.Wrap(err, "applying option")
+			}
+		}
+		return nil
+	}
+}
+
 // NewServer returns a new instance of Server.
 func NewServer(opts ...ServerOption) (*Server, error) {
 	s := &Server{

server/config.go

@@ -71,8 +71,9 @@ type Config struct {
 	// Gossip config is based around memberlist.Config.
 	Gossip gossip.Config `toml:"gossip"`
 
-	// DEPRECATED: Translation config supports translation store replication.
 	Translation struct {
+		MapSize int `toml:"map-size"`
+		// DEPRECATED: Translation config supports translation store replication.
 		PrimaryURL string `toml:"primary-url"`
 	} `toml:"translation"`
 

server/server.go

@@ -283,6 +283,16 @@ func (m *Command) SetupServer() error {
 		coordinatorOpt,
 	}
 
+	if m.Config.Translation.MapSize > 0 {
+		serverOptions = append(
+			serverOptions,
+			pilosa.OptServerHolderOptions(
+				pilosa.OptHolderTranslateFileMapSize(
+					m.Config.Translation.MapSize,
+				),
+			),
+		)
+	}
 	serverOptions = append(serverOptions, m.serverOptions...)
 
 	m.Server, err = pilosa.NewServer(serverOptions...)

test/pilosa.go

@@ -55,11 +55,22 @@ func newCommand(opts ...server.CommandOption) *Command {
 		panic(err)
 	}
 
-	// set aggressive close timeout by default to avoid hanging tests. This was
+	// Set aggressive close timeout by default to avoid hanging tests. This was
 	// a problem with PDK tests which used go-pilosa as well. We put it at the
 	// beginning of the option slice so that it can be overridden by user-passed
 	// options.
-	opts = append([]server.CommandOption{server.OptCommandCloseTimeout(time.Millisecond * 2)}, opts...)
+	// Also set TranslateFile MapSize to a smaller number so memory allocation
+	// does not fail on 32-bit systems.
+	opts = append([]server.CommandOption{
+		server.OptCommandCloseTimeout(time.Millisecond * 2),
+		server.OptCommandServerOptions(
+			pilosa.OptServerHolderOptions(
+				pilosa.OptHolderTranslateFileMapSize(
+					2 << 26,
+				),
+			),
+		),
+	}, opts...)
 	m := &Command{commandOptions: opts}
 	m.Command = server.NewCommand(bytes.NewReader(nil), ioutil.Discard, ioutil.Discard, opts...)
 	m.Config.DataDir = path

translate.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -17,6 +16,7 @@ import (
 	"time"
 
 	"github.com/cespare/xxhash"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -81,9 +81,26 @@ type TranslateFile struct {
 	replicationRetryInterval time.Duration
 }
 
+// TranslateFileOption is a functional option type for pilosa.TranslateFile
+type TranslateFileOption func(f *TranslateFile) error
+
+func OptTranslateFileMapSize(mapSize int) TranslateFileOption {
+	return func(f *TranslateFile) error {
+		f.mapSize = mapSize
+		return nil
+	}
+}
+
 // NewTranslateFile returns a new instance of TranslateFile.
-func NewTranslateFile() *TranslateFile {
-	return &TranslateFile{
+func NewTranslateFile(opts ...TranslateFileOption) *TranslateFile {
+	// 10GB default map size
+	defaultMapSize := 10 * (1 << 30)
+	// Use 2GB default map size on 32-bit systems
+	if 32<<(^uint(0)>>32&1) == 32 {
+		defaultMapSize = (1 << 31) - 1 // 2GB
+	}
+
+	f := &TranslateFile{
 		writeNotify: make(chan struct{}),
 		closing:     make(chan struct{}),
 		cols:        make(map[string]*index),
@@ -96,6 +113,16 @@ func NewTranslateFile() *TranslateFile {
 
 		replicationRetryInterval: defaultReplicationRetryInterval,
 	}
+
+	for _, opt := range opts {
+		err := opt(f)
+		if err != nil {
+			// TODO (2.0): Change func signature to return error
+			panic(errors.Wrap(err, "applying option"))
+		}
+	}
+
+	return f
 }
 
 func (s *TranslateFile) Open() (err error) {

translate_mapsize_386.go

@@ -1,5 +0,0 @@
-package pilosa
-
-// defaultMapSize is the default size of mapped memory for the translate store.
-// It is passed as an int to syscall.Mmap and so must be < 2^31
-const defaultMapSize = (1 << 31) - 1 // 2GB

translate_mapsize_all64bitsystems.go

@@ -1,8 +0,0 @@
-// +build !386
-
-package pilosa
-
-// defaultMapSize is the default size of mapped memory for the translate store.
-// It is passed as an int to syscall.Mmap and so can only be larger than 2^31 on
-// 64bit systems.
-const defaultMapSize = 10 * (1 << 30) // 10GB