scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-07 11:27:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

remove all instances of txsrc

Browse source
This commit is contained in:
Travis 2021-01-21 21:59:51 -06:00
parent 13984353e4
commit 19f91782e7
No known key found for this signature in database
GPG key ID: 37080CC2042BA34E
27 changed files with 67 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
api.go
View file

@ -1816,7 +1816,7 @@ func (api *API) Info() serverInfo {
CPUMHz: mhz,
CPUType: si.CPUModel(),
Memory: mem,
TxSrc: api.holder.txf.TxType(),
StorageBackend: api.holder.txf.TxType(),
ReplicaN: api.cluster.ReplicaN,
ShardHash: api.cluster.Hasher.Name(),
KeyHash: api.cluster.Topology.Hasher.Name(),
@ -2170,7 +2170,7 @@ type serverInfo struct {
CPUPhysicalCores int `json:"cpuPhysicalCores"`
CPULogicalCores int `json:"cpuLogicalCores"`
CPUMHz int `json:"cpuMHz"`
TxSrc string `json:"txSrc"`
StorageBackend string `json:"storageBackend"`
}
type apiMethod int

8
dbshard_internal_test.go
View file

@ -209,9 +209,9 @@ rick.index.txstores@@@/store-rbfdb@@/shard.0223-rbfdb@
`,
}
func makeSampleRoaringDir(root, index, txsrc string, minBytes int, h *Holder, view2shards *FieldView2Shards) (idx *Index) {
func makeSampleRoaringDir(root, index, backend string, minBytes int, h *Holder, view2shards *FieldView2Shards) (idx *Index) {
shards := []uint64{0, 93, 215, 217, 219, 221, 223}
fns := strings.Split(sampleRoaringDirList[txsrc], "\n")
fns := strings.Split(sampleRoaringDirList[backend], "\n")
firstDone := false
for i, fn := range fns {
@ -219,11 +219,11 @@ func makeSampleRoaringDir(root, index, txsrc string, minBytes int, h *Holder, vi
continue
}
var shard uint64
if txsrc != "roaring" {
if backend != "roaring" {
// only have shards for the non-roaring
shard = shards[i]
}
switch txsrc {
switch backend {
case "bolt", "rbf":
idx = helperCreateDBShard(h, index, shard)

6
holder.go
View file

@ -149,9 +149,9 @@ type HolderOpts struct {
// about fragments when opening them.
Inspect bool
// Txsrc controls the tx/storage engine we instatiate. Set by
// StorageBackend controls the tx/storage engine we instatiate. Set by
// server.go OptServerStorageConfig
Txsrc string
StorageBackend string
// RowcacheOn, if true, turns on the row cache for all storage backends.
RowcacheOn bool
@ -268,7 +268,7 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
OpenIDAllocator: cfg.OpenIDAllocator,
translationSyncer: cfg.TranslationSyncer,
Logger: cfg.Logger,
Opts: HolderOpts{Txsrc: cfg.StorageConfig.Backend, RowcacheOn: cfg.RowcacheOn},
Opts: HolderOpts{StorageBackend: cfg.StorageConfig.Backend, RowcacheOn: cfg.RowcacheOn},
SnapshotQueue: defaultSnapshotQueue,

2
holder_internal_test.go
View file

@ -260,7 +260,7 @@ func TestHolderOperatorCancel(t *testing.T) {
func mustHolderConfig() *HolderConfig {
cfg := DefaultHolderConfig()
if backend := CurrentBackend(); backend != "" {
_ = MustTxsrcToTxtype(backend)
_ = MustBackendToTxtype(backend)
cfg.StorageConfig.Backend = backend
}
return cfg

10
http/client_test.go
View file

@ -1422,15 +1422,15 @@ func makeImportColumnAttrsRequest(index string, shard int64, attrKey string) *pi
}
}
// verify that serverInfo has TxSrc
func TestClient_ServerInfoHasTxSrc(t *testing.T) {
// verify that serverInfo has Backend
func TestClient_ServerInfoHasBackend(t *testing.T) {
//srcs := []string{"roaring", "rbf", "lmdb"}
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
cmd := cluster.GetNode(0)
si := cmd.API.Info()
if si.TxSrc == "" {
panic("should have gotten a TxSrc back")
if si.StorageBackend == "" {
panic("should have gotten a StorageBackend back")
}
pilosa.MustTxsrcToTxtype(si.TxSrc) // panics if invalid
pilosa.MustBackendToTxtype(si.StorageBackend) // panics if invalid
}

4
http/handler.go
View file

@ -820,14 +820,14 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
req, ok := qreq.(*pilosa.QueryRequest)
if DoPerQueryProfiling {
txsrc := pilosa.CurrentBackend()
backend := pilosa.CurrentBackend()
reqHash := hash(req.Query)
qlen := len(req.Query)
if qlen > 100 {
qlen = 100
}
name := "_query." + reqHash + "." + txsrc + "." + time.Now().Format("20060102150405") + "." + req.Query[:qlen]
name := "_query." + reqHash + "." + backend + "." + time.Now().Format("20060102150405") + "." + req.Query[:qlen]
f, err := os.Create(name)
if err != nil {
panic(err)

10
pilosa.go
View file

@ -21,6 +21,7 @@ import (
"time"
pnet "github.com/pilosa/pilosa/v2/net"
"github.com/pilosa/pilosa/v2/storage"
"github.com/pkg/errors"
)
@ -224,3 +225,12 @@ func AddressWithDefaults(addr string) (*pnet.URI, error) {
func CurrentBackend() string {
return os.Getenv("PILOSA_STORAGE_BACKEND")
}
// CurrentBackendOrDefault tries the environment variable first, but falls back
// to the default backed if the environment variable is empty.
func CurrentBackendOrDefault() string {
if backend := os.Getenv("PILOSA_STORAGE_BACKEND"); backend != "" {
return backend
}
return storage.DefaultBackend
}

16
pprof.go
View file

@ -31,11 +31,11 @@ import (
// commented out—in holder.go.
func CPUProfileForDur(dur time.Duration, outpath string) {
// per-query pprof output:
txsrc := CurrentBackend()
if txsrc == "" {
txsrc = storage.DefaultBackend
backend := CurrentBackend()
if backend == "" {
backend = storage.DefaultBackend
}
path := outpath + "." + txsrc
path := outpath + "." + backend
f, err := os.Create(path)
panicOn(err)
@ -57,11 +57,11 @@ func CPUProfileForDur(dur time.Duration, outpath string) {
// commented out—in holder.go.
func MemProfileForDur(dur time.Duration, outpath string) {
// per-query pprof output:
txsrc := CurrentBackend()
if txsrc == "" {
txsrc = storage.DefaultBackend
backend := CurrentBackend()
if backend == "" {
backend = storage.DefaultBackend
}
path := outpath + "." + txsrc
path := outpath + "." + backend
f, err := os.Create(path)
panicOn(err)

4
scripts/bench_read.sh
View file

@ -29,10 +29,10 @@ do
# Execute RBF/Roaring benchmark.
RBF_PATH=gloat/data/query/${TYPE}/rbf/${DATE}.tar.gz
TXSRC=rbf gloat run -v -o "$RBF_PATH" $WORKFLOW_PATH
STORAGE_BACKEND=rbf gloat run -v -o "$RBF_PATH" $WORKFLOW_PATH
ROARING_PATH=gloat/data/query/${TYPE}/roaring/${DATE}.tar.gz
TXSRC=roaring gloat run -v -o "$ROARING_PATH" $WORKFLOW_PATH
STORAGE_BACKEND=roaring gloat run -v -o "$ROARING_PATH" $WORKFLOW_PATH
# Generate graph from results.
gloat graph -layout 2,5 -size 5120,820 -title "$TITLE" -name utime,stime,heap_alloc,heap_inuse,heap_objects,num_gc,rchar,wchar,syscr,syscw -series rbf,roaring -o /tmp/output.png $RBF_PATH $ROARING_PATH

4
scripts/bench_write.sh
View file

@ -27,10 +27,10 @@ TITLE="RBF vs Roaring, $WORKFLOW_NAME, $DATE ($SHA)"
# Execute RBF/Roaring benchmark.
RBF_PATH=gloat/data/1m/rbf/${DATE}.tar.gz
TXSRC=rbf gloat run -v -o $RBF_PATH $WORKFLOW_PATH
STORAGE_BACKEND=rbf gloat run -v -o $RBF_PATH $WORKFLOW_PATH
ROARING_PATH=gloat/data/1m/roaring/${DATE}.tar.gz
TXSRC=roaring gloat run -v -o $ROARING_PATH $WORKFLOW_PATH
STORAGE_BACKEND=roaring gloat run -v -o $ROARING_PATH $WORKFLOW_PATH
# Generate graph from results.
gloat graph -layout 2,5 -size 5120,820 -title "$TITLE" -name utime,stime,heap_alloc,heap_inuse,heap_objects,num_gc,rchar,wchar,syscr,syscw -series rbf,roaring -o /tmp/output.png $RBF_PATH $ROARING_PATH

2
scripts/etc/gloat/gh.1d.yml
View file

@ -1,6 +1,6 @@
name: "GitHub Import Load Testing (1 day)"
main: "pilosa server --data-dir ${TMPDIR} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ${TMPDIR} --storage.backend ${STORAGE_BACKEND}"
load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-01T23:00:00Z --cache-dir .githubarchive"
health_url: "http://localhost:10101/status"

2
scripts/etc/gloat/gh.1m.yml
View file

@ -1,6 +1,6 @@
name: "GitHub Import Load Testing (1 month)"
main: "pilosa server --data-dir ${TMPDIR} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ${TMPDIR} --storage.backend ${STORAGE_BACKEND}"
load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-31T23:00:00Z --cache-dir .githubarchive"
health_url: "http://localhost:10101/status"

2
scripts/etc/gloat/gh.1w.yml
View file

@ -1,6 +1,6 @@
name: "GitHub Import Load Testing (1 week)"
main: "pilosa server --data-dir ${TMPDIR} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ${TMPDIR} --storage.backend ${STORAGE_BACKEND}"
load: "molecula-consumer-github -i events -d id --record-type event --batch-size=100000 --start-time 2020-01-01T00:00:00Z --end-time 2020-01-06T23:00:00Z --cache-dir .githubarchive"
health_url: "http://localhost:10101/status"

2
scripts/etc/gloat/query.count.yml
View file

@ -1,6 +1,6 @@
name: "Count() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ~/pilosa.query.${STORAGE_BACKEND} --storage.backend ${STORAGE_BACKEND}"
load: "pilosa-bench -type count -rate 100 -n 3000"
health_url: "http://localhost:10101/status"

2
scripts/etc/gloat/query.difference.yml
View file

@ -1,6 +1,6 @@
name: "Difference() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ~/pilosa.query.${STORAGE_BACKEND} --storage.backend ${STORAGE_BACKEND}"
load: "pilosa-bench -type difference -rate 10 -n 300"
health_url: "http://localhost:10101/status"

2
scripts/etc/gloat/query.groupby.yml
View file

@ -1,6 +1,6 @@
name: "GroupBy() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ~/pilosa.query.${STORAGE_BACKEND} --storage.backend ${STORAGE_BACKEND}"
load: "pilosa-bench -type groupby -rate 100 -n 3000"
health_url: "http://localhost:10101/status"

2
scripts/etc/gloat/query.intersect.yml
View file

@ -1,6 +1,6 @@
name: "Intersect() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ~/pilosa.query.${STORAGE_BACKEND} --storage.backend ${STORAGE_BACKEND}"
load: "pilosa-bench -type intersect -rate 100 -n 3000"
health_url: "http://localhost:10101/status"

2
scripts/etc/gloat/query.row-bsi.yml
View file

@ -1,6 +1,6 @@
name: "Row(BSI) Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ~/pilosa.query.${STORAGE_BACKEND} --storage.backend ${STORAGE_BACKEND}"
load: "pilosa-bench -type row -rate 100 -n 3000"
health_url: "http://localhost:10101/status"

2
scripts/etc/gloat/query.row-range.yml
View file

@ -1,6 +1,6 @@
name: "Time-based Row() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ~/pilosa.query.${STORAGE_BACKEND} --storage.backend ${STORAGE_BACKEND}"
load: "pilosa-bench -type row-range -rate 10 -n 300 -from 2020-01-01T00:00:00Z -to 2020-01-31T23:00:00Z"
health_url: "http://localhost:10101/status"

2
scripts/etc/gloat/query.row.yml
View file

@ -1,6 +1,6 @@
name: "Row() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ~/pilosa.query.${STORAGE_BACKEND} --storage.backend ${STORAGE_BACKEND}"
load: "pilosa-bench -type row -rate 100 -n 3000"
health_url: "http://localhost:10101/status"

2
scripts/etc/gloat/query.topk.yml
View file

@ -1,6 +1,6 @@
name: "Time-based TopK() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ~/pilosa.query.${STORAGE_BACKEND} --storage.backend ${STORAGE_BACKEND}"
load: "pilosa-bench -type row-range -rate 10 -n 300 -from 2020-01-01T00:00:00Z -to 2020-01-31T23:00:00Z"
health_url: "http://localhost:10101/status"

2
scripts/etc/gloat/query.union.yml
View file

@ -1,6 +1,6 @@
name: "Union() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ~/pilosa.query.${STORAGE_BACKEND} --storage.backend ${STORAGE_BACKEND}"
load: "pilosa-bench -type union -rate 10 -n 300"
health_url: "http://localhost:10101/status"

2
scripts/etc/gloat/query.xor.yml
View file

@ -1,6 +1,6 @@
name: "Xor() Load Testing"
main: "pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC}"
main: "pilosa server --data-dir ~/pilosa.query.${STORAGE_BACKEND} --storage.backend ${STORAGE_BACKEND}"
load: "pilosa-bench -type xor -rate 10 -n 300"
health_url: "http://localhost:10101/status"

6
scripts/populate_query_db.sh
View file

@ -4,15 +4,15 @@ set -e
# This script generates data query load testing to be run against.
#
# Environment variables:
# - TXSRC: Transaction store type ("roaring", "rbf")
# - STORAGE_BACKEND: Transaction store type ("roaring", "rbf")
# - CACHEDIR: Path to local GitHub Archive data, if available.
# Require environment variables.
: "${TXSRC:?Must set TXSRC environment variable}"
: "${STORAGE_BACKEND:?Must set STORAGE_BACKEND environment variable}"
: "${GHCACHEDIR:''}"
echo "Starting pilosa"
pilosa server --data-dir ~/pilosa.query.${TXSRC} --txsrc ${TXSRC} & pid_pilosa=$!
pilosa server --data-dir ~/pilosa.query.${STORAGE_BACKEND} --storage.backend ${STORAGE_BACKEND} & pid_pilosa=$!
sleep 5
echo ""

4
stattx.go
View file

@ -69,8 +69,8 @@ func (w *callStats) reset() {
}
func (c *callStats) report() (r string) {
txsrc := CurrentBackend()
r = fmt.Sprintf("callStats: (%v)\n", txsrc)
backend := CurrentBackend()
r = fmt.Sprintf("callStats: (%v)\n", backend)
c.mu.Lock()
defer c.mu.Unlock()
var lines []*debugstats.LineSorter

3
test/cluster.go
View file

@ -464,7 +464,6 @@ func prependOpts(opts [][]server.CommandOption, size int) [][]server.CommandOpti
opts[i] = prependTestServerOpts([]server.CommandOption{})
}
} else if len(opts) == 1 {
println("len opts == 1, size = ", size)
opts2 := make([][]server.CommandOption, size)
for i := 0; i < size; i++ {
opts2[i] = prependTestServerOpts(opts[0])
@ -485,7 +484,7 @@ func prependTestServerOpts(opts []server.CommandOption) []server.CommandOption {
pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore),
pilosa.OptServerNodeDownRetries(5, 100*time.Millisecond),
pilosa.OptServerStorageConfig(&storage.Config{
Backend: pilosa.CurrentBackend(),
Backend: pilosa.CurrentBackendOrDefault(),
FsyncEnabled: true,
}),
),

19
txfactory.go
View file

@ -468,16 +468,15 @@ func (txf *TxFactory) NeedsSnapshot() (b bool) {
return
}
func MustTxsrcToTxtype(txsrc string) (types []txtype) {
func MustBackendToTxtype(backend string) (types []txtype) {
var srcs []string
if strings.Contains(txsrc, "_") {
srcs = strings.Split(txsrc, "_")
if strings.Contains(backend, "_") {
srcs = strings.Split(backend, "_")
if len(srcs) != 2 {
panic("only two blue-green comparisons permitted")
}
} else {
srcs = append(srcs, txsrc)
srcs = append(srcs, backend)
}
for i, s := range srcs {
@ -489,11 +488,11 @@ func MustTxsrcToTxtype(txsrc string) (types []txtype) {
case BoltTxn: // "bolt"
types = append(types, boltTxn)
default:
panic(fmt.Sprintf("unknown txsrc '%v'", s))
panic(fmt.Sprintf("unknown backend '%v'", s))
}
if i == 1 {
if types[1] == types[0] {
panic(fmt.Sprintf("cannot blue-green the same txsrc on both arms: '%v'", s))
panic(fmt.Sprintf("cannot blue-green the same backend on both arms: '%v'", s))
}
}
}
@ -503,12 +502,12 @@ func MustTxsrcToTxtype(txsrc string) (types []txtype) {
// NewTxFactory always opens an existing database. If you
// want to a fresh database, os.RemoveAll on dir/name ahead of time.
// We always store files in a subdir of holderDir.
func NewTxFactory(txsrc string, holderDir string, holder *Holder) (f *TxFactory, err error) {
types := MustTxsrcToTxtype(txsrc)
func NewTxFactory(backend string, holderDir string, holder *Holder) (f *TxFactory, err error) {
types := MustBackendToTxtype(backend)
f = &TxFactory{
types: types,
typeOfTx: txsrc,
typeOfTx: backend,
holder: holder,
}
if len(types) == 2 {