From 7fe7ed907f51e75f54cb81567558bc6954ac16e2 Mon Sep 17 00:00:00 2001 From: Jason Aten Date: Mon, 20 Jul 2020 17:08:38 -0400 Subject: [PATCH] env PILOSA_TXSRC is reinjected into the env if pilosa server --tx overrides it. fixes #575 1. Tests can choose the Tx engine desired by setting the PILOSA_TXSRC env variable. For example: PILOSA_TXSRC=badger go test -v -run TestImportClearRestart 2. pilosa server --tx is enabled now. Examples: pilosa server --tx roaring # gives the legacy approach. pilosa server --tx rbf # will activate RBF pilosa server --tx badger # will activate BadgerDB pilosa server --tx badger_rbf # will run Blue-Green badger to RBF comparisons. and so forth. See pilosa server -h or txfactory.go for all valid --tx choices. 3. Mechanism that makes both tests(1) and pilosa server(2) work at once: pilosa/server/server.go injects PILOSA_TXSRC into env to communicate with NewIndex in pilosa/index.go. --- Makefile | 3 +++ ctl/server.go | 2 +- index.go | 18 +++++++++++++++++- server/server.go | 24 ++++++++++++++++++++++++ txfactory.go | 5 +++-- 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 566ddaf53..e1577b62b 100644 --- a/Makefile +++ b/Makefile @@ -178,6 +178,9 @@ golangci-lint: require-golangci-lint # Alias linter: golangci-lint +# Better alias +ocd: golangci-lint + # Run gometalinter with custom flags gometalinter: require-gometalinter vendor GO111MODULE=off gometalinter --vendor --disable-all \ diff --git a/ctl/server.go b/ctl/server.go index aa0b9a077..6dc4eb0a6 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -87,5 +87,5 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/ of events.") // Transactional storage engine - flags.StringVarP(&srv.Config.Txsrc, "tx", "", "roaring", "transaction/storage to use: one of roaring, rbf, badger, rbf_roaring, roaring_rbf, badger_roaring, roaring_badger, badger_rbf, or rbf_badger") + flags.StringVarP(&srv.Config.Txsrc, "tx", "", "", "transaction/storage to use: one of roaring, rbf, badger, rbf_roaring, roaring_rbf, badger_roaring, roaring_badger, badger_rbf, or rbf_badger (default roaring)") } diff --git a/index.go b/index.go index b1e2a03b2..7361df02c 100644 --- a/index.go +++ b/index.go @@ -78,9 +78,25 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) { // Emulate what the spf13/cobra does, letting env vars override // the defaults, because we may be under a simple "go test" run where // not all that command line machinery has been spun up. + + // needed for the tests: txsrc := os.Getenv("PILOSA_TXSRC") + + // Warning: won't work for the tests to say: + // txsrc := holder.Opts.Txsrc // WILL BREAK TESTS + if txsrc == "" { - txsrc = DefaultTxsrc + // nothing in the env for PILOSA_TXSRC; therefore not running under a "make topt.badger" for example. + if holder.Opts.Txsrc != "" { + // most of the tests and production run, we expect that if holder.opts.Txsrc is set, it + // will be the exact same as PILOSA_TXSRC. Unfortunately there are tests where that won't hold. + // So if the env var PILOSA_TXSRC *is* set, we always give it precedence. + // This lets `PILOSA_TXSRC=rbf go test -v -run "one_of_my_RBF_tests"` succeed. + // + txsrc = holder.Opts.Txsrc + } else { + txsrc = DefaultTxsrc + } } txf, err := newTxFactory(txsrc, path) if err != nil { diff --git a/server/server.go b/server/server.go index df78a33bf..2b1cea4bb 100644 --- a/server/server.go +++ b/server/server.go @@ -240,6 +240,29 @@ func (m *Command) SetupServer() error { m.logger.Printf("%s", pilosa.VersionInfo()) + // If the pilosa command line uses -tx to override the + // PILOSA_TXSRC env variable, then we must also correct + // the environment, so that pilosa/txfactory.go can determine the + // desired Tx engine. This enables "go test" testing in pilosa that + // does not spin up a full server, while still respecting the pilosa + // server's choice when run full in production. + envTxsrc := os.Getenv("PILOSA_TXSRC") + if m.Config.Txsrc == "" { + // INVAR: No -tx flag on the command line. + // We defer to the environment, and then the DefaultTxsrc + if envTxsrc == "" { + // no env variable requested either. + m.Config.Txsrc = pilosa.DefaultTxsrc + } else { + // Tell the "regular" prod server what to use. + m.Config.Txsrc = envTxsrc + } + } + // INVAR: m.Config.Txsrc is valid and not "", but pilosa.DefaultTxsrc could be bad. + txty := pilosa.MustTxsrcToTxtype(m.Config.Txsrc) // will panic on unknown Txsrc. + os.Setenv("PILOSA_TXSRC", m.Config.Txsrc) + m.logger.Printf("using Txsrc '%v'/%v", m.Config.Txsrc, txty) + // validateAddrs sets the appropriate values for Bind and Advertise // based on the inputs. It is not responsible for applying defaults, although // it does provide a non-zero port (10101) in the case where no port is specified. @@ -353,6 +376,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts), pilosa.OptServerSerializer(proto.Serializer{}), + pilosa.OptServerTxsrc(m.Config.Txsrc), coordinatorOpt, } diff --git a/txfactory.go b/txfactory.go index 6d31f43ce..f8af55c6c 100644 --- a/txfactory.go +++ b/txfactory.go @@ -85,7 +85,7 @@ const ( blueGreenRBFBadger txtype = 9 ) -func txsrcToTxtype(txsrc string) txtype { +func MustTxsrcToTxtype(txsrc string) txtype { switch txsrc { case RoaringTxn: // "roaring" return roaringFragmentFilesTxn @@ -110,10 +110,11 @@ func txsrcToTxtype(txsrc string) txtype { } func newTxFactory(txsrc string, path string) (f *TxFactory, err error) { - ty := txsrcToTxtype(txsrc) + ty := MustTxsrcToTxtype(txsrc) if ty < 1 || ty > 9 { panic(fmt.Sprintf("invalid txtype '%v'", int(ty))) } + var bw *BadgerDBWrapper if ty == badgerTxn || ty == 4 || ty == 5 || ty == 8 || ty == 9 { bw, err = openBadgerDBWrapper(path)