env PILOSA_TXSRC is reinjected into the env if pilosa server --tx overrides it. fixes #575

1. Tests can choose the Tx engine desired by setting the PILOSA_TXSRC
      env variable. For example:

         PILOSA_TXSRC=badger go test -v -run TestImportClearRestart

   2. pilosa server --tx is enabled now.

     Examples:
      pilosa server --tx roaring    # gives the legacy approach.
      pilosa server --tx rbf        # will activate RBF
      pilosa server --tx badger     # will activate BadgerDB
      pilosa server --tx badger_rbf # will run Blue-Green badger to RBF comparisons.

      and so forth. See pilosa server -h or txfactory.go for all valid --tx choices.

   3. Mechanism that makes both tests(1) and pilosa server(2) work at once:
       pilosa/server/server.go injects PILOSA_TXSRC into env to
       communicate with NewIndex in pilosa/index.go.
This commit is contained in:
Jason Aten 2020-07-20 17:08:38 -04:00
parent 22fd24566a
commit 7fe7ed907f
5 changed files with 48 additions and 4 deletions

View file

@ -178,6 +178,9 @@ golangci-lint: require-golangci-lint
# Alias
linter: golangci-lint
# Better alias
ocd: golangci-lint
# Run gometalinter with custom flags
gometalinter: require-gometalinter vendor
GO111MODULE=off gometalinter --vendor --disable-all \

View file

@ -87,5 +87,5 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/<rate> of events.")
// Transactional storage engine
flags.StringVarP(&srv.Config.Txsrc, "tx", "", "roaring", "transaction/storage to use: one of roaring, rbf, badger, rbf_roaring, roaring_rbf, badger_roaring, roaring_badger, badger_rbf, or rbf_badger")
flags.StringVarP(&srv.Config.Txsrc, "tx", "", "", "transaction/storage to use: one of roaring, rbf, badger, rbf_roaring, roaring_rbf, badger_roaring, roaring_badger, badger_rbf, or rbf_badger (default roaring)")
}

View file

@ -78,9 +78,25 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
// Emulate what the spf13/cobra does, letting env vars override
// the defaults, because we may be under a simple "go test" run where
// not all that command line machinery has been spun up.
// needed for the tests:
txsrc := os.Getenv("PILOSA_TXSRC")
// Warning: won't work for the tests to say:
// txsrc := holder.Opts.Txsrc // WILL BREAK TESTS
if txsrc == "" {
txsrc = DefaultTxsrc
// nothing in the env for PILOSA_TXSRC; therefore not running under a "make topt.badger" for example.
if holder.Opts.Txsrc != "" {
// most of the tests and production run, we expect that if holder.opts.Txsrc is set, it
// will be the exact same as PILOSA_TXSRC. Unfortunately there are tests where that won't hold.
// So if the env var PILOSA_TXSRC *is* set, we always give it precedence.
// This lets `PILOSA_TXSRC=rbf go test -v -run "one_of_my_RBF_tests"` succeed.
//
txsrc = holder.Opts.Txsrc
} else {
txsrc = DefaultTxsrc
}
}
txf, err := newTxFactory(txsrc, path)
if err != nil {

View file

@ -240,6 +240,29 @@ func (m *Command) SetupServer() error {
m.logger.Printf("%s", pilosa.VersionInfo())
// If the pilosa command line uses -tx to override the
// PILOSA_TXSRC env variable, then we must also correct
// the environment, so that pilosa/txfactory.go can determine the
// desired Tx engine. This enables "go test" testing in pilosa that
// does not spin up a full server, while still respecting the pilosa
// server's choice when run full in production.
envTxsrc := os.Getenv("PILOSA_TXSRC")
if m.Config.Txsrc == "" {
// INVAR: No -tx flag on the command line.
// We defer to the environment, and then the DefaultTxsrc
if envTxsrc == "" {
// no env variable requested either.
m.Config.Txsrc = pilosa.DefaultTxsrc
} else {
// Tell the "regular" prod server what to use.
m.Config.Txsrc = envTxsrc
}
}
// INVAR: m.Config.Txsrc is valid and not "", but pilosa.DefaultTxsrc could be bad.
txty := pilosa.MustTxsrcToTxtype(m.Config.Txsrc) // will panic on unknown Txsrc.
os.Setenv("PILOSA_TXSRC", m.Config.Txsrc)
m.logger.Printf("using Txsrc '%v'/%v", m.Config.Txsrc, txty)
// validateAddrs sets the appropriate values for Bind and Advertise
// based on the inputs. It is not responsible for applying defaults, although
// it does provide a non-zero port (10101) in the case where no port is specified.
@ -353,6 +376,7 @@ func (m *Command) SetupServer() error {
pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts),
pilosa.OptServerSerializer(proto.Serializer{}),
pilosa.OptServerTxsrc(m.Config.Txsrc),
coordinatorOpt,
}

View file

@ -85,7 +85,7 @@ const (
blueGreenRBFBadger txtype = 9
)
func txsrcToTxtype(txsrc string) txtype {
func MustTxsrcToTxtype(txsrc string) txtype {
switch txsrc {
case RoaringTxn: // "roaring"
return roaringFragmentFilesTxn
@ -110,10 +110,11 @@ func txsrcToTxtype(txsrc string) txtype {
}
func newTxFactory(txsrc string, path string) (f *TxFactory, err error) {
ty := txsrcToTxtype(txsrc)
ty := MustTxsrcToTxtype(txsrc)
if ty < 1 || ty > 9 {
panic(fmt.Sprintf("invalid txtype '%v'", int(ty)))
}
var bw *BadgerDBWrapper
if ty == badgerTxn || ty == 4 || ty == 5 || ty == 8 || ty == 9 {
bw, err = openBadgerDBWrapper(path)