diff --git a/ctl/server.go b/ctl/server.go index 07fdf887a..c72f7d379 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -18,7 +18,6 @@ import ( "fmt" "time" - "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/server" "github.com/pilosa/pilosa/v2/storage" "github.com/spf13/cobra" @@ -106,7 +105,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // Transactional storage engine // Note: the default for --tx must be kept "" empty string. Otherwise we // cannot detect and honor the PILOSA_TXSRC env var over-ride. - flags.StringVarP(&srv.Config.Txsrc, "txsrc", "", "", fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_TXSRC is over-ridden by --txsrc option on the command line.", pilosa.DefaultTxsrc)) + flags.StringVarP(&srv.Config.Txsrc, "txsrc", "", "", fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_TXSRC is over-ridden by --txsrc option on the command line.", storage.DefaultBackend)) // Storage // Note: the default for --storage.backend must be kept "" empty string. diff --git a/executor_test.go b/executor_test.go index 906d0d276..8c5323303 100644 --- a/executor_test.go +++ b/executor_test.go @@ -540,7 +540,7 @@ func TestExecutor_Execute_Count(t *testing.T) { func roaringOnlyTest(t *testing.T) { src := os.Getenv("PILOSA_TXSRC") - if src == pilosa.RoaringTxn || (pilosa.DefaultTxsrc == pilosa.RoaringTxn && src == "") { + if src == pilosa.RoaringTxn || (storage.DefaultBackend == pilosa.RoaringTxn && src == "") { // okay to run, we are under roaring only } else { t.Skip("skip for everything but roaring") diff --git a/fragment_internal_test.go b/fragment_internal_test.go index fa45aace4..780a6d0b7 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -38,6 +38,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/storage" "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -1720,7 +1721,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { func roaringOnlyTest(t *testing.T) { src := os.Getenv("PILOSA_TXSRC") - if src == RoaringTxn || (DefaultTxsrc == RoaringTxn && src == "") { + if src == RoaringTxn || (storage.DefaultBackend == RoaringTxn && src == "") { // okay to run, we are under roaring only } else { t.Skip("skip for everything but roaring") @@ -1729,7 +1730,7 @@ func roaringOnlyTest(t *testing.T) { func roaringOnlyBenchmark(b *testing.B) { src := os.Getenv("PILOSA_TXSRC") - if src == RoaringTxn || (DefaultTxsrc == RoaringTxn && src == "") { + if src == RoaringTxn || (storage.DefaultBackend == RoaringTxn && src == "") { // okay to run, we are under roaring only } else { b.Skip("skip for everything but roaring") diff --git a/pprof.go b/pprof.go index 5c9b0b339..a13ef62d2 100644 --- a/pprof.go +++ b/pprof.go @@ -22,14 +22,18 @@ import ( "time" _ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server. + + "github.com/pilosa/pilosa/v2/storage" ) +// CPUProfileForDur (where "Dur" is short for "Duration"), is used for +// performance tuning during development. It's only called—but is currently +// commented out—in holder.go. func CPUProfileForDur(dur time.Duration, outpath string) { - // per-query pprof output: txsrc := os.Getenv("PILOSA_TXSRC") if txsrc == "" { - txsrc = DefaultTxsrc + txsrc = storage.DefaultBackend } path := outpath + "." + txsrc f, err := os.Create(path) @@ -48,12 +52,14 @@ func CPUProfileForDur(dur time.Duration, outpath string) { }() } +// MemProfileForDur (where "Dur" is short for "Duration"), is used for +// performance tuning during development. It's only called—but is currently +// commented out—in holder.go. func MemProfileForDur(dur time.Duration, outpath string) { - // per-query pprof output: txsrc := os.Getenv("PILOSA_TXSRC") if txsrc == "" { - txsrc = DefaultTxsrc + txsrc = storage.DefaultBackend } path := outpath + "." + txsrc f, err := os.Create(path) diff --git a/server/server.go b/server/server.go index 0709aeddf..702f57b30 100644 --- a/server/server.go +++ b/server/server.go @@ -55,6 +55,7 @@ import ( "github.com/pilosa/pilosa/v2/statik" "github.com/pilosa/pilosa/v2/stats" "github.com/pilosa/pilosa/v2/statsd" + "github.com/pilosa/pilosa/v2/storage" "github.com/pilosa/pilosa/v2/syswrap" "github.com/pilosa/pilosa/v2/testhook" "github.com/pkg/errors" @@ -291,17 +292,17 @@ func (m *Command) SetupServer() error { envTxsrc := os.Getenv("PILOSA_TXSRC") if m.Config.Txsrc == "" { // INVAR: No -tx flag on the command line. - // We defer to the environment, and then the DefaultTxsrc + // We defer to the environment, and then the DefaultBackend if envTxsrc == "" { // no env variable requested either. - m.Config.Txsrc = pilosa.DefaultTxsrc + m.Config.Txsrc = storage.DefaultBackend } else { // Tell the "regular" prod server what to use. m.Config.Txsrc = envTxsrc } } - // INVAR: m.Config.Txsrc is valid and not "", but pilosa.DefaultTxsrc could be bad. - txty := pilosa.MustTxsrcToTxtype(m.Config.Txsrc) // will panic on unknown Txsrc. + // INVAR: m.Config.Storage.Backend is valid and not "", but storage.DefaultBackend could be bad. + txty := pilosa.MustTxsrcToTxtype(m.Config.Storage.Backend) // will panic on unknown Backend. os.Setenv("PILOSA_TXSRC", m.Config.Txsrc) m.logger.Printf("using Txsrc '%v'/%v", m.Config.Txsrc, txty) if len(txty) == 2 { diff --git a/stattx.go b/stattx.go index 02b60d6e1..8004e795f 100644 --- a/stattx.go +++ b/stattx.go @@ -24,9 +24,9 @@ import ( "sync" "time" + "github.com/pilosa/pilosa/v2/debugstats" "github.com/pilosa/pilosa/v2/roaring" txkey "github.com/pilosa/pilosa/v2/short_txkey" - //txkey "github.com/pilosa/pilosa/v2/txkey" ) // statTx is useful to profile on a @@ -69,29 +69,12 @@ func (w *callStats) reset() { } } -type LineSorter struct { - Line string - Tot float64 -} - -type SortByTot []*LineSorter - -func (p SortByTot) Len() int { - return len(p) -} -func (p SortByTot) Less(i, j int) bool { - return p[i].Tot < p[j].Tot -} -func (p SortByTot) Swap(i, j int) { - p[i], p[j] = p[j], p[i] -} - func (c *callStats) report() (r string) { txsrc := os.Getenv("PILOSA_TXSRC") r = fmt.Sprintf("callStats: (%v)\n", txsrc) c.mu.Lock() defer c.mu.Unlock() - var lines []*LineSorter + var lines []*debugstats.LineSorter for i := kall(0); i < kLast; i++ { slc := c.elap[i].dur n := len(slc) @@ -105,9 +88,9 @@ func (c *callStats) report() (r string) { totaltm = slc[0] } line := fmt.Sprintf(" %20v N=%8v avg/op: %12v sd: %12v total: %12v\n", i.String(), n, time.Duration(mean), time.Duration(sd), time.Duration(totaltm)) - lines = append(lines, &LineSorter{Line: line, Tot: totaltm}) + lines = append(lines, &debugstats.LineSorter{Line: line, Tot: totaltm}) } - sort.Sort(SortByTot(lines)) + sort.Sort(debugstats.SortByTot(lines)) for i := range lines { r += lines[i].Line } diff --git a/tx_test.go b/tx_test.go index e9a74090c..b942651b2 100644 --- a/tx_test.go +++ b/tx_test.go @@ -24,6 +24,7 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/server" + "github.com/pilosa/pilosa/v2/storage" "github.com/pilosa/pilosa/v2/test" ) @@ -61,9 +62,9 @@ func queryBalances(m0api *pilosa.API, acctOwnerID uint64, fldAcct0, fldAcct1, in func skipForRoaring(t *testing.T) { src := os.Getenv("PILOSA_TXSRC") - // once txfactory.go DefaultTxsrc != RoaringTxn, this + // once txfactory.go storage.DefaultBackend != RoaringTxn, this // will break, of course. Take out the src == "" below. - if (src == "" && pilosa.DefaultTxsrc == pilosa.RoaringTxn) || strings.Contains(src, "roaring") { + if (src == "" && storage.DefaultBackend == pilosa.RoaringTxn) || strings.Contains(src, "roaring") { t.Skip("skip if roaring pseudo-txn involved -- won't show transactional rollback") } } diff --git a/txfactory.go b/txfactory.go index 92018c03f..818bb8ac3 100644 --- a/txfactory.go +++ b/txfactory.go @@ -31,7 +31,6 @@ import ( "github.com/pilosa/pilosa/v2/roaring" txkey "github.com/pilosa/pilosa/v2/short_txkey" "github.com/pilosa/pilosa/v2/storage" - //txkey "github.com/pilosa/pilosa/v2/txkey" "github.com/pkg/errors" "github.com/zeebo/blake3" ) @@ -43,11 +42,6 @@ const ( BoltTxn string = "bolt" ) -// DefaultTxsrc is set here. pilosa/server/config.go references it -// to set the default for pilosa server exeutable. -// Can be overridden with env variable PILOSA_TXSRC for testing. -const DefaultTxsrc = RoaringTxn - // DetectMemAccessPastTx true helps us catch places in api and executor // where mmapped memory is being accessed after the point in time // which the transaction has committed or rolled back. Since