scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-12 15:51:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #997 from molecula/bolt_in_badger_out

Browse source 
Use boltdb instead of badger as our all-Go Tx oracle
This commit is contained in:
tgruben 2020-10-16 17:41:23 -05:00 • committed by GitHub
commit 092177aea4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 1845 additions and 2685 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
Makefile
View file

@ -215,17 +215,17 @@ topt:
@echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
@echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l
topt-badger:
mv log.topt.badger log.topt.badger.prev || true
PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.badger
@echo " log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l
@echo " log.topt.badger red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l
topt-bolt:
mv log.topt.bolt log.topt.bolt.prev || true
PILOSA_TXSRC=bolt go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.bolt
@echo " log.topt.bolt green: \c"; cat log.topt.bolt | grep PASS |wc -l
@echo " log.topt.bolt red: \c"; cat log.topt.bolt | grep '\-\-\- FAIL' |wc -l
topt-badger-race:
mv log.topt.badger-race log.topt.badger-race.prev || true
PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.badger-race
@echo " log.topt.badger-race green: \c"; cat log.topt.badger-race | grep PASS |wc -l
@echo " log.topt.badger-race red: \c"; cat log.topt.badger-race | grep '\-\-\- FAIL' |wc -l
topt-bolt-race:
mv log.topt.bolt-race log.topt.bolt-race.prev || true
PILOSA_TXSRC=bolt go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.bolt-race
@echo " log.topt.bolt-race green: \c"; cat log.topt.bolt-race | grep PASS |wc -l
@echo " log.topt.bolt-race red: \c"; cat log.topt.bolt-race | grep '\-\-\- FAIL' |wc -l
topt-rbf:
mv log.topt.rbf log.topt.rbf.prev || true
@ -257,20 +257,20 @@ topt-race:
@echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
@echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l
# blue-green checks. These run two different storage engines (rbf, roaring, or badger)
# blue-green checks. These run two different storage engines (rbf, roaring, or bolt)
# and compare each transaction for a result.
bg-rr: # shorthand for bluegreen test with A:badger; B:roaring
mv log.bg-rr log.bg-rr.prev || true
PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.bg-rr
@echo " log.bg-rr green: \c"; cat log.bg-rr | grep PASS |wc -l
@echo " log.bg-rr red: \c"; cat log.bg-rr | grep '\-\-\- FAIL' |wc -l
bt-rr: # shorthand for bluegreen test with A:bolt; B:roaring
mv log.bt-rr log.bt-rr.prev || true
PILOSA_TXSRC=bolt_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.bt-rr
@echo " log.bt-rr green: \c"; cat log.bt-rr | grep PASS |wc -l
@echo " log.bt-rr red: \c"; cat log.bt-rr | grep '\-\-\- FAIL' |wc -l
rr-bg: # bluegreen with A:roaring; B:badger (B's values are returned).
mv log.bg.roar_bg log.bg.roar_bg.prev || true
PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rr-bg
@echo " log.rr-bg green: \c"; cat log.rr-bg | grep PASS |wc -l
@echo " log.rr-bg red: \c"; cat log.rr-bg | grep '\-\-\- FAIL' |wc -l
rr-bt: # bluegreen with A:roaring; B:bolt (B's values are returned).
mv log.bt.roar_bt log.bt.roar_bt.prev || true
PILOSA_TXSRC=roaring_bolt go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rr-bt
@echo " log.rr-bt green: \c"; cat log.rr-bt | grep PASS |wc -l
@echo " log.rr-bt red: \c"; cat log.rr-bt | grep '\-\-\- FAIL' |wc -l
rbf-rr:
mv log.rbf-rr log.rbf-rr.prev || true
@ -284,17 +284,17 @@ rr-rbf:
@echo " log.rr-rbf green: \c"; cat log.rr-rbf | grep PASS |wc -l
@echo " log.rr-rbf red: \c"; cat log.rr-rbf | grep '\-\-\- FAIL' |wc -l
rbf-bg:
mv log.rbf-bg log.rbf-bg.prev || true
PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rbf-bg
@echo " log.rbf-bg green: \c"; cat log.rbf-bg | grep PASS |wc -l
@echo " log.rbf-bg red: \c"; cat log.rbf-bg | grep '\-\-\- FAIL' |wc -l
rbf-bt:
mv log.rbf-bt log.rbf-bt.prev || true
PILOSA_TXSRC=rbf_bolt go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rbf-bt
@echo " log.rbf-bt green: \c"; cat log.rbf-bt | grep PASS |wc -l
@echo " log.rbf-bt red: \c"; cat log.rbf-bt | grep '\-\-\- FAIL' |wc -l
bg-rbf:
mv log.bg-rbf log.bg-rbf.prev || true
PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.bg-rbf
@echo " log.bg-rbf green: \c"; cat log.bg-rbf | grep PASS |wc -l
@echo " log.bg-rbf red: \c"; cat log.bg-rbf | grep '\-\-\- FAIL' |wc -l
bt-rbf:
mv log.bt-rbf log.bt-rbf.prev || true
PILOSA_TXSRC=bolt_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.bt-rbf
@echo " log.bt-rbf green: \c"; cat log.bt-rbf | grep PASS |wc -l
@echo " log.bt-rbf red: \c"; cat log.bt-rbf | grep '\-\-\- FAIL' |wc -l
rbf-lm:
mv log.rbf-lm log.rbf-lm.prev || true
@ -320,17 +320,17 @@ rr-lm:
@echo " log.rr-lm green: \c"; cat log.rr-lm | grep PASS |wc -l
@echo " log.rr-lm red: \c"; cat log.rr-lm | grep '\-\-\- FAIL' |wc -l
bg-lm:
mv log.topt.bg-lm log.topt.bg-lm.prev || true
PILOSA_TXSRC=badger_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.bg-lm
@echo " log.topt.bg-lm green: \c"; cat log.topt.bg-lm | grep PASS |wc -l
@echo " log.topt.bg-lm red: \c"; cat log.topt.bg-lm | grep '\-\-\- FAIL' |wc -l
bt-lm:
mv log.topt.bt-lm log.topt.bt-lm.prev || true
PILOSA_TXSRC=bolt_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.bt-lm
@echo " log.topt.bt-lm green: \c"; cat log.topt.bt-lm | grep PASS |wc -l
@echo " log.topt.bt-lm red: \c"; cat log.topt.bt-lm | grep '\-\-\- FAIL' |wc -l
lm-bg:
mv log.topt.lm-bg log.topt.lm-bg.prev || true
PILOSA_TXSRC=lmdb_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.lm-bg
@echo " log.topt.lm-bg green: \c"; cat log.topt.lm-bg | grep PASS |wc -l
@echo " log.topt.lm-bg red: \c"; cat log.topt.lm-bg | grep '\-\-\- FAIL' |wc -l
lm-bt:
mv log.topt.lm-bt log.topt.lm-bt.prev || true
PILOSA_TXSRC=lmdb_bolt go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.lm-bt
@echo " log.topt.lm-bt green: \c"; cat log.topt.lm-bt | grep PASS |wc -l
@echo " log.topt.lm-bt red: \c"; cat log.topt.lm-bt | grep '\-\-\- FAIL' |wc -l
# Run golangci-lint

3
api.go
View file

@ -655,7 +655,8 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
// Ensure data is flushed.
cw.Flush()
span.LogKV("n", n)
return tx.Commit()
tx.Rollback()
return nil
}
// ShardNodes returns the node and all replicas which should contain a shard's data.

1924
badger.go
View file

File diff suppressed because it is too large Load diff

1564
bolt.go Normal file
View file

File diff suppressed because it is too large Load diff

793
badger_test.go → bolt_test.go
View file

File diff suppressed because it is too large Load diff

14
dbshard.go
View file

@ -376,7 +376,7 @@ func (dbs *DBShard) DumpAll() {
case roaringTxn:
case rbfTxn:
case lmdbTxn:
case badgerTxn:
case boltTxn:
default:
panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
}
@ -410,7 +410,13 @@ func (dbs *DBShard) pathForType(ty txtype) string {
// what here for roaring? well, roaringRegistrar.OpenDBWrapper()
// is a no-op anyhow. so doesn't need to be correct atm.
return dbs.HolderPath + sep + dbs.Index + ".index.txstores@@@" + sep + "store" + ty.FileSuffix() + "@" + sep + fmt.Sprintf("shard.%04v%v", dbs.Shard, ty.FileSuffix())
path := dbs.HolderPath + sep + dbs.Index + ".index.txstores@@@" + sep + "store" + ty.FileSuffix() + "@" + sep + fmt.Sprintf("shard.%04v%v", dbs.Shard, ty.FileSuffix())
if ty == boltTxn {
// special case:
// bolt doesn't use a directory like the others, just a direct path.
path += sep + "bolt.db"
}
return path
}
// if you don't know the shard, you have to use this.
@ -466,8 +472,8 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *
registry = globalRbfDBReg
case lmdbTxn:
registry = globalLMDBReg
case badgerTxn:
registry = globalBadgerReg
case boltTxn:
registry = globalBoltReg
default:
panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
}

26
dbshard_internal_test.go
View file

@ -74,7 +74,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
orig := os.Getenv("PILOSA_TXSRC")
defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
for _, src := range []string{"lmdb", "roaring", "badger", "rbf"} {
for _, src := range []string{"lmdb", "roaring", "bolt", "rbf"} {
os.Setenv("PILOSA_TXSRC", src)
@ -149,13 +149,13 @@ rick.index.txstores@@@/store-lmdb@@/shard.0219-lmdb@
rick.index.txstores@@@/store-lmdb@@/shard.0221-lmdb@
rick.index.txstores@@@/store-lmdb@@/shard.0223-lmdb@
`,
"badger": `
rick.index.txstores@@@/store-badgerdb@@/shard.0093-badgerdb@
rick.index.txstores@@@/store-badgerdb@@/shard.0215-badgerdb@
rick.index.txstores@@@/store-badgerdb@@/shard.0217-badgerdb@
rick.index.txstores@@@/store-badgerdb@@/shard.0219-badgerdb@
rick.index.txstores@@@/store-badgerdb@@/shard.0221-badgerdb@
rick.index.txstores@@@/store-badgerdb@@/shard.0223-badgerdb@
"bolt": `
rick.index.txstores@@@/store-boltdb@@/shard.0093-boltdb@/bolt.db
rick.index.txstores@@@/store-boltdb@@/shard.0215-boltdb@/bolt.db
rick.index.txstores@@@/store-boltdb@@/shard.0217-boltdb@/bolt.db
rick.index.txstores@@@/store-boltdb@@/shard.0219-boltdb@/bolt.db
rick.index.txstores@@@/store-boltdb@@/shard.0221-boltdb@/bolt.db
rick.index.txstores@@@/store-boltdb@@/shard.0223-boltdb@/bolt.db
`,
"rbf": `
rick.index.txstores@@@/store-rbfdb@@/shard.0093-rbfdb@
@ -189,8 +189,8 @@ func makeSampleRoaringDir(root, txsrc string, minBytes int, h *Holder) {
// DBPerShard won't know anything about it.
helperCreateDBShard(h, index, shard)
continue
case "badger":
makeBadgertestDB(root+sep+fn, h, shard)
case "bolt":
makeBolttestDB(root+sep+fn, h, shard)
helperCreateDBShard(h, index, shard)
continue
case "rbf":
@ -227,10 +227,10 @@ func makeLMDBtestDB(path string, h *Holder, shard uint64) {
}
func makeBadgertestDB(path string, h *Holder, shard uint64) {
func makeBolttestDB(path string, h *Holder, shard uint64) {
i := uint64(1)
w, _ := mustOpenEmptyBadgerWrapper(path)
badgerDBMustSetBitvalue(w, "index", "field", "view", shard, i)
w, _ := mustOpenEmptyBoltWrapper(path)
BoltMustSetBitvalue(w, "index", "field", "view", shard, i)
w.Close()
}

10
dbshard_test.go
View file

@ -35,15 +35,7 @@ func skipForNonLMDB(t *testing.T) {
}
}
var _ = skipForNonLMDB // happy linter
var _ = skipForNonBadger // happy linter
func skipForNonBadger(t *testing.T) {
src := os.Getenv("PILOSA_TXSRC")
if src != "badger" {
t.Skip("skip if not badger")
}
}
var _ = skipForNonLMDB // happy linter
// Can't write it all to one shard like we do (did).
func Test_DBPerShard_multiple_shards_used(t *testing.T) {

1
executor.go
View file

@ -3138,7 +3138,6 @@ func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string,
// Handle bsiGroup ranges differently.
if c.HasConditionArg() {
// looks the same on badger/roaring. we think.
return e.executeRowBSIGroupShard(ctx, qcx, index, c, shard)
}

2
field_internal_test.go
View file

@ -914,6 +914,6 @@ func TestBSIGroup_TxReopenDB(t *testing.T) {
panicOn(qcx.Finish())
} // loop
// the test: can we re-open a BSI fragment under badger/rbf.
// the test: can we re-open a BSI fragment under Tx store
_ = f.Reopen()
}

22
fragment_internal_test.go
View file

@ -560,7 +560,7 @@ func TestFragment_Sum(t *testing.T) {
}
})
panicOn(tx.Commit())
tx.Rollback()
tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -2178,7 +2178,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) {
}
}
panicOn(tx.Commit())
tx.Rollback()
tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -2457,7 +2457,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) {
}
}
panicOn(tx.Commit())
tx.Rollback()
tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -2707,7 +2707,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) {
}
}
panicOn(tx.Commit())
tx.Rollback()
tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -2787,7 +2787,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
if err := f.bulkImport(tx, rows, cols, options); err != nil {
b.Fatalf("Error Building Sample: %s", err)
}
panicOn(tx.Commit())
tx.Rollback()
if row > max {
max = row
}
@ -2949,7 +2949,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) {
// the cost of actually doing the op log for the large initial data set
// is excessive. force storage into snapshotted state, then use import
// to generate an op log and/or snapshot.
// note: skipped for badger, above.
// note: skipped for rbf, bolt, lmdb, above.
_, _, err := frags[j].storage.ImportRoaringBits(data, false, false, 0)
if err != nil {
b.Fatalf("importing roaring: %v", err)
@ -3376,7 +3376,7 @@ func BenchmarkFileWrite(b *testing.B) {
/////////////////////////////////////////////////////////////////////
// not called under badger b/c f.idx.NeedsSnapshot() in Clean() avoids it.
// not called under Tx stores b/c f.idx.NeedsSnapshot() in Clean() avoids it.
func (f *fragment) sanityCheck(t testing.TB) {
newBM := roaring.NewFileBitmap()
file, err := os.Open(f.path)
@ -3393,7 +3393,7 @@ func (f *fragment) sanityCheck(t testing.TB) {
t.Fatalf("sanityCheck couldn't unmarshal fragment %s: %v", f.path, err)
}
// Refactor fragment.storage
// note: not called for badger, see above.
// note: not called for rbf, see above.
if equal, reason := newBM.BitwiseEqual(f.storage); !equal {
t.Fatalf("fragment %s: unmarshalled bitmap different: %v", f.path, reason)
}
@ -3408,11 +3408,11 @@ func (f *fragment) Clean(t testing.TB) {
// check or else, in some cases, the background snapshot queue
// can decide to pick it up.
func() {
// should we skip snapshot queue stuff under badger/rbf?
// should we skip snapshot queue stuff under bolt/rbf?
defer f.mu.Unlock()
// badger doesn't need snapshot, so this stuff is skipped.
// The snapshot queue stuff doesn't work under badger.
// rbf doesn't need snapshot, so this stuff is skipped.
// The snapshot queue stuff doesn't work under rbf.
if f.idx.NeedsSnapshot() {
err := f.holder.SnapshotQueue.Await(f)
if err != nil {

3
go.mod
View file

@ -11,7 +11,6 @@ require (
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
github.com/davecgh/go-spew v1.1.1
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361
github.com/dustin/go-humanize v1.0.0
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311
github.com/glycerine/lmdb-go v1.9.34
@ -36,6 +35,7 @@ require (
github.com/satori/go.uuid v1.2.0
github.com/shirou/gopsutil v2.18.12+incompatible
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.4.0
@ -50,6 +50,7 @@ require (
golang.org/x/text v0.3.3 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/grpc v1.28.0
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
modernc.org/mathutil v1.0.0
modernc.org/strutil v1.0.0
vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible

19
go.sum
View file

@ -5,8 +5,6 @@ github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWu
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE=
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ=
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
@ -33,24 +31,16 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I=
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE=
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 h1:JBNM90aGLCiF9iJYvpvayMpYeW498v5ZDZqE2chqZ2A=
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE=
github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA=
github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@ -88,8 +78,6 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
@ -208,7 +196,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qq
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@ -231,7 +218,6 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
@ -240,7 +226,6 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -258,7 +243,6 @@ github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW
github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0=
@ -278,7 +262,6 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@ -320,11 +303,9 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

2
holder_test.go
View file

@ -808,7 +808,7 @@ func TestHolderSyncer_IntField(t *testing.T) {
t.Fatalf("syncing node 1: %v", err)
}
// dump the badger keys for both c0 and c1
// dump the rbf keys for both c0 and c1
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {

14
lmdb.go
View file

@ -39,6 +39,8 @@ import (
"github.com/pkg/errors"
)
const TxInitialMmapSize = 4 << 30 // 4GB
// Linux builds note:
//
// This setting of -DMDB_USE_SYSV_SEM=1 is required in the cgo build
@ -160,7 +162,7 @@ func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper
err = env.SetMaxDBs(1)
panicOn(err)
//err = env.SetMapSize(256 << 30) // 256GB
err = env.SetMapSize(4 << 30) // 4GB
err = env.SetMapSize(TxInitialMmapSize)
panicOn(err)
panicOn(os.MkdirAll(filepath.Dir(path), 0755))
@ -563,8 +565,8 @@ const isDebugRun = false
// Rollback rolls back the transaction.
func (tx *LMDBTx) Rollback() {
alreadyDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1)
if !alreadyDone {
notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1)
if !notDone {
return
}
//vv("lmdb rollback tx _sn_ %v; stack \n%v", tx.sn) // , stack())
@ -590,8 +592,8 @@ func (tx *LMDBTx) Rollback() {
// Commits can handle up to 100k updates to fragments
// at once, but not more. This is a LMDBDB imposed limit.
func (tx *LMDBTx) Commit() error {
alreadyDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1)
if !alreadyDone {
notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1)
if !notDone {
return nil
}
//vv("lmdb commit tx _sn_ %v; stack \n%v", tx.sn, stack())
@ -892,7 +894,7 @@ type LMDBIterator struct {
}
// NewLMDBIterator creates an iterator on tx that will
// only return badgerKeys that start with prefix.
// only return rbf that start with prefix.
func NewLMDBIterator(tx *LMDBTx, prefix []byte) (bi *LMDBIterator) {
tx.mu.Lock()

9
lmdb_other.go
View file

@ -31,6 +31,13 @@ import (
var _ = time.Now
const isDebugRun = false
const TxInitialMmapSize = 1 << 30 // 1GB
func ToContainer(typ byte, w []byte) (c *roaring.Container) {
panic("ToContainer not implemented yet on non-amd64")
}
// lmdbRegistrar facilitates shutdown
// of all the lmdb databases started under
// tests. Its needed because most tests don't cleanup
@ -252,7 +259,7 @@ func (tx *LMDBTx) ContainerIterator(index, field, view string, shard uint64, fir
type LMDBIterator struct{}
// NewLMDBIterator creates an iterator on tx that will
// only return badgerKeys that start with prefix.
// only return lmdbKeys that start with prefix.
func NewLMDBIterator(tx *LMDBTx, prefix []byte) (bi *LMDBIterator) {
panic("lmdb only available on 64-bit arch")
}

8
lmdb_test.go
View file

@ -932,12 +932,6 @@ func TestLMDB_RoaringBitmap(t *testing.T) {
}
}
// no reverse iterator on LMDB; we did a special case for Max
// rather than a general purpose reverse iterator which we
// aren't using for anything else.
//func TestLMDB_reverse_badger_iterator_and_prefix_valid(t *testing.T)
//func TestLMDB_just_reverse_badger_iterator_and_prefix_valid(t *testing.T)
func TestLMDB_ImportRoaringBits(t *testing.T) {
dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_ImportRoaringBits")
@ -946,7 +940,7 @@ func TestLMDB_ImportRoaringBits(t *testing.T) {
index, field, view, shard := "i", "f", "v", uint64(0)
tx, _ := dbwrap.NewTx(writable, index, Txo{})
defer tx.Rollback()
tx.(*LMDBTx).DeleteEmptyContainer = true // traditional badger Tx behavior, but not Roaring.
tx.(*LMDBTx).DeleteEmptyContainer = true // match Roaring
//bitvalue := uint64(42)

2
rrtx.go
View file

@ -575,7 +575,7 @@ func (w *RoaringWrapper) DeleteField(index, field, fieldPath string) error {
// match txn sn count vs lmdb/etc.
atomic.AddInt64(&globalNextTxSnRoaring, 1)
// under blue-green badger_roaring, the directory will not be found, b/c badger will have
// under blue-green bolt_roaring, the directory will not be found, b/c bolt will have
// already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs:
// "If the path does not exist, RemoveAll returns nil (no error)"
err := os.RemoveAll(fieldPath)

2
tournament.sh
View file

@ -5,7 +5,7 @@
## one or two backends through the rigors of make testv-race.
## logs are saved to the tourna.log.${i} files.
for i in rbf lmdb roaring rbf_lmdb rbf_roaring lmdb_rbf lmdb_roaring roaring_rbf roaring_lmdb ; do
for i in rbf lmdb roaring bolt rbf_lmdb rbf_roaring lmdb_rbf lmdb_roaring roaring_rbf roaring_lmdb roaring_bolt lmdb_bolt; do
echo "$(date) starting ${i}, output to tourna.log.${i}"
echo "***=== ${i} ====================*** $(date)" &> tourna.log.${i}
PILOSA_TXSRC=${i} make testv-race 2>&1 > tourna.log.${i}

2
tx.go
View file

@ -48,7 +48,7 @@ const writable = true
// that have not been committed.
type Tx interface {
// Type returns "roaring", "rbf", "badger", "badger_roaring", or one of the other
// Type returns "roaring", "rbf", "bolt", "badger_roaring", or one of the other
// blue-green Tx types at the top of txfactory.go
Type() string

20
txfactory.go
View file

@ -38,7 +38,7 @@ const (
RoaringTxn string = "roaring"
LmdbTxn string = "lmdb"
RBFTxn string = "rbf"
BadgerTxn string = "badger"
BoltTxn string = "bolt"
)
// DefaultTxsrc is set here. pilosa/server/config.go references it
@ -409,12 +409,12 @@ const (
roaringTxn txtype = 1 // these don't really have any transactions
rbfTxn txtype = 2
lmdbTxn txtype = 3
badgerTxn txtype = 4
boltTxn txtype = 4
)
// these need to be skipped by the holder.go field scanner that
// calls IsTxDatabasePath
var allTypesWithSuffixes = []txtype{rbfTxn, lmdbTxn, badgerTxn}
var allTypesWithSuffixes = []txtype{rbfTxn, lmdbTxn, boltTxn}
// FileSuffix is used to determine backend directory names.
// We append '@' to be sure we never collide with a field name
@ -429,8 +429,8 @@ func (ty txtype) FileSuffix() string {
return "-rbfdb@"
case lmdbTxn:
return "-lmdb@"
case badgerTxn:
return "-badgerdb@"
case boltTxn:
return "-boltdb@"
}
panic(fmt.Sprintf("unkown txtype %v", int(ty)))
}
@ -478,8 +478,8 @@ func MustTxsrcToTxtype(txsrc string) (types []txtype) {
types = append(types, rbfTxn)
case LmdbTxn: // "lmdb"
types = append(types, lmdbTxn)
case BadgerTxn: // "badger"
types = append(types, badgerTxn)
case BoltTxn: // "bolt"
types = append(types, boltTxn)
default:
panic(fmt.Sprintf("unknown txsrc '%v'", s))
}
@ -844,8 +844,8 @@ func (ty txtype) String() string {
return "rbfTxn"
case lmdbTxn:
return "lmdbTxn"
case badgerTxn:
return "badgerTxn"
case boltTxn:
return "boltTxn"
}
panic(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty)))
}
@ -1212,7 +1212,7 @@ func anyGlobalDBWrappersStillOpen() bool {
if globalLMDBReg.Size() != 0 {
return true
}
if globalBadgerReg.Size() != 0 {
if globalBoltReg.Size() != 0 {
return true
}
return false

6
txfactory_internal_test.go
View file

@ -26,10 +26,10 @@ import (
func Test_TxFactory_Qcx_query_context(t *testing.T) {
src := os.Getenv("PILOSA_TXSRC")
if src == "rbf" || src == "lmdb" || src == "badger" {
if src == "rbf" || src == "lmdb" || src == "bolt" {
// ok
} else {
t.Skip("this test only for lmdb and rbf and badger")
t.Skip("this test only for lmdb and rbf and bolt")
}
shard := uint64(0)
@ -251,7 +251,7 @@ func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) {
orig := os.Getenv("PILOSA_TXSRC")
defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
checked := []string{"lmdb", "roaring", "badger", "rbf"}
checked := []string{"lmdb", "roaring", "bolt", "rbf"}
for _, blue := range checked {
for _, green := range checked {

2
utils_internal_test.go
View file

@ -497,7 +497,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
// this is the *test* version of a network call, transferring fragments between
// nodes in a cluster. So it is allowed to be kind of a hack.
// there will be two -badgerdb directories/databases, we need to copy
// there will be two -rbfdb directories/databases, we need to copy
// from src to dest the fragment. This simulates sending the fragment over the network.
srcIdx := srcCluster.holder.Index(src.Index)
srctx := srcIdx.holder.txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment, Shard: srcFragment.shard})