diff --git a/Makefile b/Makefile index 3f1a42949..e3e29ded7 100644 --- a/Makefile +++ b/Makefile @@ -215,17 +215,17 @@ topt: @echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l @echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l -topt-badger: - mv log.topt.badger log.topt.badger.prev || true - PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.badger - @echo " log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l - @echo " log.topt.badger red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l +topt-bolt: + mv log.topt.bolt log.topt.bolt.prev || true + PILOSA_TXSRC=bolt go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.bolt + @echo " log.topt.bolt green: \c"; cat log.topt.bolt | grep PASS |wc -l + @echo " log.topt.bolt red: \c"; cat log.topt.bolt | grep '\-\-\- FAIL' |wc -l -topt-badger-race: - mv log.topt.badger-race log.topt.badger-race.prev || true - PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.badger-race - @echo " log.topt.badger-race green: \c"; cat log.topt.badger-race | grep PASS |wc -l - @echo " log.topt.badger-race red: \c"; cat log.topt.badger-race | grep '\-\-\- FAIL' |wc -l +topt-bolt-race: + mv log.topt.bolt-race log.topt.bolt-race.prev || true + PILOSA_TXSRC=bolt go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.bolt-race + @echo " log.topt.bolt-race green: \c"; cat log.topt.bolt-race | grep PASS |wc -l + @echo " log.topt.bolt-race red: \c"; cat log.topt.bolt-race | grep '\-\-\- FAIL' |wc -l topt-rbf: mv log.topt.rbf log.topt.rbf.prev || true @@ -257,20 +257,20 @@ topt-race: @echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l @echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l -# blue-green checks. These run two different storage engines (rbf, roaring, or badger) +# blue-green checks. These run two different storage engines (rbf, roaring, or bolt) # and compare each transaction for a result. -bg-rr: # shorthand for bluegreen test with A:badger; B:roaring - mv log.bg-rr log.bg-rr.prev || true - PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.bg-rr - @echo " log.bg-rr green: \c"; cat log.bg-rr | grep PASS |wc -l - @echo " log.bg-rr red: \c"; cat log.bg-rr | grep '\-\-\- FAIL' |wc -l +bt-rr: # shorthand for bluegreen test with A:bolt; B:roaring + mv log.bt-rr log.bt-rr.prev || true + PILOSA_TXSRC=bolt_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.bt-rr + @echo " log.bt-rr green: \c"; cat log.bt-rr | grep PASS |wc -l + @echo " log.bt-rr red: \c"; cat log.bt-rr | grep '\-\-\- FAIL' |wc -l -rr-bg: # bluegreen with A:roaring; B:badger (B's values are returned). - mv log.bg.roar_bg log.bg.roar_bg.prev || true - PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rr-bg - @echo " log.rr-bg green: \c"; cat log.rr-bg | grep PASS |wc -l - @echo " log.rr-bg red: \c"; cat log.rr-bg | grep '\-\-\- FAIL' |wc -l +rr-bt: # bluegreen with A:roaring; B:bolt (B's values are returned). + mv log.bt.roar_bt log.bt.roar_bt.prev || true + PILOSA_TXSRC=roaring_bolt go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rr-bt + @echo " log.rr-bt green: \c"; cat log.rr-bt | grep PASS |wc -l + @echo " log.rr-bt red: \c"; cat log.rr-bt | grep '\-\-\- FAIL' |wc -l rbf-rr: mv log.rbf-rr log.rbf-rr.prev || true @@ -284,17 +284,17 @@ rr-rbf: @echo " log.rr-rbf green: \c"; cat log.rr-rbf | grep PASS |wc -l @echo " log.rr-rbf red: \c"; cat log.rr-rbf | grep '\-\-\- FAIL' |wc -l -rbf-bg: - mv log.rbf-bg log.rbf-bg.prev || true - PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rbf-bg - @echo " log.rbf-bg green: \c"; cat log.rbf-bg | grep PASS |wc -l - @echo " log.rbf-bg red: \c"; cat log.rbf-bg | grep '\-\-\- FAIL' |wc -l +rbf-bt: + mv log.rbf-bt log.rbf-bt.prev || true + PILOSA_TXSRC=rbf_bolt go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rbf-bt + @echo " log.rbf-bt green: \c"; cat log.rbf-bt | grep PASS |wc -l + @echo " log.rbf-bt red: \c"; cat log.rbf-bt | grep '\-\-\- FAIL' |wc -l -bg-rbf: - mv log.bg-rbf log.bg-rbf.prev || true - PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.bg-rbf - @echo " log.bg-rbf green: \c"; cat log.bg-rbf | grep PASS |wc -l - @echo " log.bg-rbf red: \c"; cat log.bg-rbf | grep '\-\-\- FAIL' |wc -l +bt-rbf: + mv log.bt-rbf log.bt-rbf.prev || true + PILOSA_TXSRC=bolt_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.bt-rbf + @echo " log.bt-rbf green: \c"; cat log.bt-rbf | grep PASS |wc -l + @echo " log.bt-rbf red: \c"; cat log.bt-rbf | grep '\-\-\- FAIL' |wc -l rbf-lm: mv log.rbf-lm log.rbf-lm.prev || true @@ -320,17 +320,17 @@ rr-lm: @echo " log.rr-lm green: \c"; cat log.rr-lm | grep PASS |wc -l @echo " log.rr-lm red: \c"; cat log.rr-lm | grep '\-\-\- FAIL' |wc -l -bg-lm: - mv log.topt.bg-lm log.topt.bg-lm.prev || true - PILOSA_TXSRC=badger_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.bg-lm - @echo " log.topt.bg-lm green: \c"; cat log.topt.bg-lm | grep PASS |wc -l - @echo " log.topt.bg-lm red: \c"; cat log.topt.bg-lm | grep '\-\-\- FAIL' |wc -l +bt-lm: + mv log.topt.bt-lm log.topt.bt-lm.prev || true + PILOSA_TXSRC=bolt_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.bt-lm + @echo " log.topt.bt-lm green: \c"; cat log.topt.bt-lm | grep PASS |wc -l + @echo " log.topt.bt-lm red: \c"; cat log.topt.bt-lm | grep '\-\-\- FAIL' |wc -l -lm-bg: - mv log.topt.lm-bg log.topt.lm-bg.prev || true - PILOSA_TXSRC=lmdb_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.lm-bg - @echo " log.topt.lm-bg green: \c"; cat log.topt.lm-bg | grep PASS |wc -l - @echo " log.topt.lm-bg red: \c"; cat log.topt.lm-bg | grep '\-\-\- FAIL' |wc -l +lm-bt: + mv log.topt.lm-bt log.topt.lm-bt.prev || true + PILOSA_TXSRC=lmdb_bolt go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.lm-bt + @echo " log.topt.lm-bt green: \c"; cat log.topt.lm-bt | grep PASS |wc -l + @echo " log.topt.lm-bt red: \c"; cat log.topt.lm-bt | grep '\-\-\- FAIL' |wc -l # Run golangci-lint diff --git a/api.go b/api.go index 31813ae65..924cad009 100644 --- a/api.go +++ b/api.go @@ -655,7 +655,8 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin // Ensure data is flushed. cw.Flush() span.LogKV("n", n) - return tx.Commit() + tx.Rollback() + return nil } // ShardNodes returns the node and all replicas which should contain a shard's data. diff --git a/badger.go b/badger.go deleted file mode 100644 index f667da77c..000000000 --- a/badger.go +++ /dev/null @@ -1,1924 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "log" - "math" - "os" - "sort" - "strings" - "sync" - "sync/atomic" - "time" - - badger "github.com/dgraph-io/badger/v2" - badgeroptions "github.com/dgraph-io/badger/v2/options" - "github.com/pilosa/pilosa/v2/hash" - "github.com/pilosa/pilosa/v2/rbf" - "github.com/pilosa/pilosa/v2/roaring" - "github.com/pilosa/pilosa/v2/testhook" - "github.com/pilosa/pilosa/v2/txkey" - "github.com/pkg/errors" -) - -// TODO: is there a more optimal time to do badger garbage collection? -// As in: do we need to be more aggressive about cleaning in -// proportion to write activity? Space monitoring available with the -// badger.DB.Size() (lsm, vlog int64) call. -// -// See: https://godoc.org/github.com/dgraph-io/badger#DB.RunValueLogGC -// and: https://github.com/dgraph-io/badger#garbage-collection -// -// For now we run GC periodically every 1 minute or as set by the -// BadgerDBWrapper.GcEveryDur duration. -// -// Background: (quoting from docs referenced above) -// -// "Badger relies on the client to perform garbage collection at a time of -// their choosing. It provides the following method, which can be invoked -// at an appropriate time: -// -// "DB.RunValueLogGC(): This method is designed to do garbage collection while -// Badger is online. Along with randomly picking a file, it uses statistics -// generated by the LSM-tree compactions to pick files that are likely to -// lead to maximum space reclamation. It is recommended to be called during -// periods of low activity in your system, or periodically. One call would -// only result in removal of at max one log file. As an optimization, you -// could also immediately re-run it whenever it returns nil error (indicating -// a successful value log GC), as shown below." -// -// ticker := time.NewTicker(5 * time.Minute) -// defer ticker.Stop() -// for range ticker.C { -// again: -// err := db.RunValueLogGC(0.5) -// if err == nil { -// goto again -// } -// } -// - -// -// ========================================================= -// A note on using a recent version of badgerdb: -// -// We require a v2 release of badger after 2020 May 13, when support for -// multiple read-write iterators within one transaction was added. -// Many executor_test.go tests do foreachRow() operations, -// which call BadgerTx.ContainerIterator(), which in turn creates -// a first read-write iterator, and then OffsetRange(), which needs a -// second iterator, while still in the same read-write transaction. -// -// The most recent v2 master was pulled in and added to go.mod -// by doing go get github.com/dgraph-io/badger/v2@master -// resulting in the go.mod line -// github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 -// as of this writing, 2020 July 09. This version contains the support -// for having multiple read-write iterators. -// -// Reference on github.com/dgraph-io/badger -// -// commit af22dfd8d51317d765f0c05dcdf1d15981cca4f3 -// Author: Elliot Courant -// Date: Wed May 13 01:07:33 2020 -0500 -// -// Support multiple iterators in read-write transactions. (#1286) -// -// This adds support for multiple iterators during a read-write transaction. The -// iterators created in a read-write transaction will only be able to see writes -// that were performed before the iterator was created. Any writes that occur -// after the iterator is created will be invisible to the iterator. -// -// Fixes https://github.com/dgraph-io/badger/issues/981 -// -// -// Otherwise we'll get these panics: -// 'Only one iterator can be active at one time, for a RW txn.' -// when trying to open a second iterator on the same write transaction. -// e.g. go test -v -run TestExecutor_TranslateRowsOnBool - -var badgerDefaultLogger *BadgerLog -var badgerTestLogger *BadgerLog - -const BadgerLogToStderr = false - -func init() { - // badger test output clutters up the screen, dump to /dev/null for now. - null, err := os.Open(os.DevNull) - panicOn(err) - var out io.Writer = null - if BadgerLogToStderr { - // view badger logs - out = os.Stderr - } - badgerTestLogger = &BadgerLog{Logger: log.New(out, "badger ", log.LstdFlags)} - badgerDefaultLogger = badgerTestLogger - - // BadgerDB recommends a minimum of 128 GOMAXPROCS to make use of the IOPs - // available on the SSD. So we set that here. Details: - // - // from https://github.com/dgraph-io/badger#are-there-any-go-specific-settings-that-i-should-use - // - // "We *highly* recommend setting a high number for GOMAXPROCS, - // which allows Go to observe the full IOPS throughput provided by - // modern SSDs. In Dgraph, we have set it to 128. For more details, - // see this thread [https://groups.google.com/forum/#!topic/golang-nuts/jPb_h3TvlKE/discussion]." - // - // From that thread on golang-nuts: - // - // "Manish Rai Jain - // 8/7/17 - // Hey folks, - // During Gophercon, I happened to meet Russ Cox and asked him the same question. - // If File::Read blocks goroutines, which then spawn new OS threads, in a long running job, - // there should be plenty of OS threads created already, so the random read throughput - // should increase over time and stabilize to the maximum possible value. But, that's - // not what I see in my benchmarks. - // - // And his explanation was that the GOMAXPROCS in a way acts like a multiplexer. - // From docs, "the GOMAXPROCS variable limits the number of operating system threads - // that can execute user-level Go code simultaneously." Which basically means, all - // reads must first be run only via GOMAXPROCS number of goroutines, before switching - // over to some OS thread (not really a switch, but conceptually speaking). This - // introduces a bottleneck for throughput. - // I re-ran my benchmarks with a much higher GOMAXPROCS and was able to then - // achieve the maximum throughput. The numbers are here: - // https://github.com/dgraph-io/badger-bench/blob/master/randread/maxprocs.txt - // To summarize these benchmarks, Linux fio achieves 118K IOPS, and with GOMAXPROCS=64/128, - // I'm able to achieve 105K IOPS, which is close enough. Win! - // - // Regarding the point about using io_submit etc., instead of goroutines; I managed to - // find a library which does that, but it performed worse than just using goroutines. - // https://github.com/traetox/goaio/issues/3 - // From what I gather (talking to Russ and Ian), whatever work is going on in user space, - // the same work has to happen in kernel space; so there's not much benefit here. - // - // Overall, with GOMAXPROCS set to a higher value (as I've done in Dgraph), one can get - // the advertised SSD throughput using goroutines." - // - // moved to holder init - //runtime.GOMAXPROCS(128) -} - -// BadgerLog exists because badger requires a particular logger interface, with a -// Debugf method that is not on standard library log.Logger -type BadgerLog struct { - *log.Logger -} - -// Errorf logs an error. -func (l *BadgerLog) Errorf(f string, v ...interface{}) { - l.Printf("ERROR: "+f, v...) -} - -// Warningf logs a warning. -func (l *BadgerLog) Warningf(f string, v ...interface{}) { - l.Printf("WARNING: "+f, v...) -} - -// Infof logs an informational statement. -func (l *BadgerLog) Infof(f string, v ...interface{}) { - l.Printf("INFO: "+f, v...) -} - -// Debugf logs a debug statement. -func (l *BadgerLog) Debugf(f string, v ...interface{}) { - l.Printf("DEBUG: "+f, v...) -} - -// badgerRegistrar facilitates shutdown -// of all the badger databases started under -// tests. Its needed because most tests don't cleanup -// the *Index(es) they create. But we still -// want to shutdown badgerDB goroutines -// after tests run. -// -// It also allows opening the same path twice to -// result in sharing the same open database handle, and -// thus the same transactional guarantees. -// -type badgerRegistrar struct { - mu sync.Mutex - mp map[*BadgerDBWrapper]bool - - path2db map[string]*BadgerDBWrapper -} - -func (r *badgerRegistrar) Size() int { - r.mu.Lock() - defer r.mu.Unlock() - nmp := len(r.mp) - npa := len(r.path2db) - if nmp != npa { - panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa)) - } - return nmp -} - -var globalBadgerReg *badgerRegistrar = newBadgerTestRegistrar() - -func newBadgerTestRegistrar() *badgerRegistrar { - return &badgerRegistrar{ - mp: make(map[*BadgerDBWrapper]bool), - path2db: make(map[string]*BadgerDBWrapper), - } -} - -// register each badger created under tests, so we -// can clean them up. This is called by openBadgerDBWrapper() while -// holding the r.mu.Lock, since it needs to atomically -// check the registry and make a new instance only -// if one does not exist for its path, and otherwise -// return the existing instance. -func (r *badgerRegistrar) unprotectedRegister(w *BadgerDBWrapper) { - r.mp[w] = true - r.path2db[w.path] = w -} - -// unregister removes w from r -func (r *badgerRegistrar) unregister(w *BadgerDBWrapper) { - r.mu.Lock() - delete(r.mp, w) - delete(r.path2db, w.path) - r.mu.Unlock() -} - -func DumpAllBadger() { - globalBadgerReg.mu.Lock() - defer globalBadgerReg.mu.Unlock() - for w := range globalBadgerReg.mp { - _ = w - AlwaysPrintf("this badger path='%v' has: \n%v\n", w.path, w.StringifiedBadgerKeys(nil, true)) - } -} - -// badgerPath is a helper for determining the full directory -// in which the badger database will be stored. -func badgerPath(path string) string { - if !strings.HasSuffix(path, "-badgerdb@") { - return path + "-badgerdb@" - } - return path -} - -// openBadgerDB opens the database in the bpath directoy -// without deleting any prior content. Any BadgerDB -// database directory will have the "-badgerdb" suffix. -// -// openBadgerDB will check the registry and make a new instance only -// if one does not exist for its bpath. Otherwise it returns -// the existing instance. This insures only one badgerDB -// per bpath in this pilosa node. -func (r *badgerRegistrar) OpenDBWrapper(bpath string, doAllocZero bool) (DBWrapper, error) { - // now that newTxFactory can call us directly, we might not - // have the -badgerdb suffix. - bpath = badgerPath(bpath) - - err := os.MkdirAll(bpath, 0755) - if err != nil { - return nil, err - } - r.mu.Lock() - defer r.mu.Unlock() - w, ok := r.path2db[bpath] - if ok { - // creates the effect of having only one badger open per pilosa node. - return w, nil - } - // otherwise, make a new badger and store it in globalBadgerReg - - // regular: works on amd64, but 386 doesn't work. - opt := badger.DefaultOptions(bpath).WithLogger(badgerDefaultLogger) - - opt.Compression = badgeroptions.None // turn off compression. - opt.ZSTDCompressionLevel = 0 // really, just in case. - opt.SyncWrites = true // default is true, safe. - //opt.KeepL0InMemory = true // speedup? - - // MaxCacheSize docs: - // - // how much data cache should hold in memory. A small size of - // cache means lower memory consumption and lookups/iterations - // would take longer. It is recommended to use a cache if you're - // using compression or encryption. If compression and - // encryption both are disabled, adding a cache will lead to - // unnecessary overhead which will affect the read performance. - // Setting size to zero disables the cache altogether. - //opt.MaxCacheSize = 1 << 30 // slows down 135 sec vs 113 sec on our benchmark - //opt.MaxCacheSize = 0 - opt.LoadBloomsOnOpen = false // should speed up start-up time. - - //opt.KeepBlocksInCache = true // default false - //opt.KeepBlockIndicesInCache = true // default false - - opt.BlockSize = 8 * 1024 // default 4 * 1024 - - // to get memory only do: - //opt := badger.DefaultOptions("").WithLogger(badgerDefaultLogger).WithInMemory(true) - - //db, err := badger.Open(opt) - db, err := badger.OpenManaged(opt) - if err != nil { - return nil, err - } - halt := make(chan bool) - w = &BadgerDBWrapper{ - reg: r, - path: bpath, - db: db, - halt: halt, - hasher: hash.NewBlake3Hasher(), - doAllocZero: doAllocZero, - } - _ = testhook.Opened(NewAuditor(), w, nil) - r.unprotectedRegister(w) - - w.startBadgerGarbageCollectionBackgroundGoro() - return w, nil -} - -func (w *BadgerDBWrapper) DeleteDBPath(dbs *DBShard) error { - path := dbs.pathForType(badgerTxn) - return os.RemoveAll(path) -} - -// DeleteIndex deletes all the containers associated with -// the named index from the badger database. -func (w *BadgerDBWrapper) DeleteIndex(indexName string) error { - - // We use the apostrophie rune `'` to locate the end of the - // index name in the key prefix, so we cannot allow indexNames - // themselves to contain apostrophies. - if strings.Contains(indexName, "'") { - return fmt.Errorf("error: bad indexName `%v` in BadgerDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes.", indexName) - } - prefix := txkey.IndexOnlyPrefix(indexName) - return w.DeletePrefix(prefix) -} - -// startBadgerGarbageCollectionBackgroundGoro handles Badger DB -// garbage colection by regularly purging the value log from -// a background goroutine. w.GcEveryDur controls how often -// it runs. The default is after every 60 seconds. -func (w *BadgerDBWrapper) startBadgerGarbageCollectionBackgroundGoro() { - go func() { - dur := w.GcEveryDur - if dur == 0 { - dur = time.Minute - } - ticker := time.NewTicker(dur) - defer ticker.Stop() - for { - select { - case <-ticker.C: - w.muGC.Lock() - again: - err := w.db.RunValueLogGC(0.5) - if err == nil { - goto again - } - w.muGC.Unlock() - case <-w.halt: - return - } - } - }() -} - -// statically confirm that BadgerTx satisfies the Tx interface. -var _ Tx = (*BadgerTx)(nil) - -var globalNextTxSnBadger int64 - -// BadgerDBWrapper provides the NewTx() method. -// The methods on BadgerDBWrapper are thread-safe, and can be called -// from different goroutines/threads. -type BadgerDBWrapper struct { - // serialize operations on BadgerDBWrapper and thus on the .db too, - // when obtaining new txns on different goroutines. - muDb sync.Mutex - - path string - db *badger.DB - - // track our registrar for Close / goro leak reporting purposes. - reg *badgerRegistrar - - // openTx and openIt are BadgerDBWrapper scoped tables of all open - // transactions and iterators. These are primarily for debugging purposes. - // openTx and openIt should only be read/written after locking the muOpenTxIt mutex. - - // the bool value is the writable attribute of the key *BadgerTx - openTx map[*BadgerTx]bool - - // the bool value is whether the iterator is reversed - openIt map[*BadgerIterator]bool - - // protect openTx and openIt - muOpenTxIt sync.Mutex - - // close(halt) to shutdown the badger gc goroutine in Close() - halt chan bool - - // make BadgerDBWrapper.Close() idempotent, avoiding panic on double Close() - closed bool - - // GcEveryDur controls how often the background goroutine - // runs garbage collection on the on-disk values-log. - // It defaults to running a GC every 1 minute if left as 0. - GcEveryDur time.Duration - - // muGC ensures we only run one Garbage Collection at a time. - muGC sync.Mutex - - hasher *hash.Blake3Hasher - - // doAllocZero sets the corresponding flag on all new BadgerTx. - // When doAllocZero is true, we zero out any data from badger - // after transcation commit and rollback. This simulates - // what would happen if we were to use the mmap-ed data - // from badger directly. Currently we copy by default for - // safety because otherwise TestAPI_ImportColumnAttrs sees - // corrupted data. - doAllocZero bool - - // stack() from our creation point, to track tests - // that haven't closed us. - startStack string - - DeleteEmptyContainer bool - - writeBatch *badger.WriteBatch -} - -func (w *BadgerDBWrapper) Path() string { - return w.path -} - -func (w *BadgerDBWrapper) SetHolder(h *Holder) { - // don't need it at the moment - //w.h = h -} - -// EntryCount for Badger returns a count of either 0 or 1. A 1 -// means it -func (w *BadgerDBWrapper) HasData() (has bool, err error) { - tx, err := w.NewTx(!writable, "", Txo{Write: false}) - if err != nil { - return false, err - } - defer tx.Rollback() - bi := NewBadgerIterator(tx.(*BadgerTx), nil) - defer bi.Close() - bi.Seek(nil) - if bi.it.Valid() { - return true, nil - } - return false, nil -} - -func (w *BadgerDBWrapper) CleanupTx(tx Tx) { - r := tx.(*BadgerTx) - r.mu.Lock() - defer r.mu.Unlock() - w.unprotectedCleanupTx(r) -} - -func (w *BadgerDBWrapper) unprotectedCleanupTx(btx *BadgerTx) { - if btx.done { - return - } - btx.done = true - btx.o.dbs.Cleanup(btx) // release the read/write lock. -} - -// unprotectedListOpenTxAsString is a debugging helper. -// It is not thread safe, but is only used for debugging. Called internally while -// holding locks. -func (w *BadgerDBWrapper) unprotectedListOpenTxAsString() (r string) { - - r = "openTx list = [" - for txn, write := range w.openTx { - r += fmt.Sprintf("txn p=%p(write:%v), ", txn, write) - } - return r + "]" -} - -var _ = (*BadgerDBWrapper)(nil).unprotectedListOpenTxAsString // linter happy - -// UnprotectedListOpenItAsString is exported because it is -// used for debugging in some of the pilosa_test tests. -// It is not thread safe, but only used for debugging. Called internally -// while holding locks and externally while not. -func (w *BadgerDBWrapper) UnprotectedListOpenItAsString() (r string) { - r = "openIt list = [" - for it, reverse := range w.openIt { - r += fmt.Sprintf("it p=%p(reverse:%v), ", it, reverse) - } - return r + "]" -} - -// NewTx produces BadgerDB based ACID transactions. If -// the transaction will modify data, then the write flag must be true. -// Read-only queries should set write to false, to allow more concurrency. -// Methods on a BadgerTx are thread-safe, and can be called from -// different goroutines. -// -// initialIndexName is optional. It is set by the TxFactory from the Txo -// options provided at the Tx creation point. It allows us to recognize -// and isolate cross-index queries more quickly. It can always be empty "" -// but when set is highly useful for debugging. It has no impact -// on transaction behavior. -// -func (w *BadgerDBWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { - - sn := atomic.AddInt64(&globalNextTxSnBadger, 1) - - tx = &BadgerTx{ - frag: o.Fragment, - write: write, - tx: w.db.NewTransactionAt(1, write), - Db: w, - doAllocZero: w.doAllocZero, - initialIndexName: initialIndexName, - DeleteEmptyContainer: w.DeleteEmptyContainer, - o: o, - sn: sn, - } - return -} - -// Close shuts down the Badger database. -func (w *BadgerDBWrapper) Close() (err error) { - - w.muDb.Lock() - defer w.muDb.Unlock() - if !w.closed { - w.reg.unregister(w) - close(w.halt) - w.closed = true - } - _ = testhook.Closed(NewAuditor(), w, nil) - return w.db.Close() -} - -// BadgerTx wraps a badger.Txn and provides the Tx interface -// method implementations. -// The methods on BadgerTx are thread-safe, and can be called -// from different goroutines. -type BadgerTx struct { - - // mu serializes badger operations on this single txn instance. - // - // reference: https://godoc.org/github.com/dgraph-io/badger - // "Running [two separate -jea] transactions concurrently is OK. However, a - // transaction itself isn't thread safe, and should only - // be run serially. It doesn't matter if a transaction is - // created by one goroutine and passed down to other, as - // long as the Txn APIs are called serially." - mu sync.Mutex - - write bool - Db *BadgerDBWrapper - tx *badger.Txn - - frag *fragment - opcount int - - doAllocZero bool - - // for tracking txn boundary issues, track all the memory - // that we deploy for roaring containers, and zero it on - // transaction commit/rollback. - acMu sync.Mutex // protect ourAllocs and ourContainers - ourAllocs [][]byte - ourContainers []*roaring.Container - - initialIndexName string - - DeleteEmptyContainer bool - - // We must avoid writing more than 10MB to badger in - // one transaction. If we go over, then - // we'll get a ErrTxnTooBig error. At that point - // we can't commit more, because the transaction - // will "conflict". So we must monitor - // totals written and auto-commit before going - // over the limits to avoid wedging into an - // unrecoverable state. - // - // We now use the approach described in https://github.com/dgraph-io/badger/issues/539 - // wherein "I fixed it by using a ManagedDB and using a commtTs of 1 always." - // But see also - // https://github.com/dgraph-io/badger/issues/409 - // wrt to "Write skew anomaly with concurrent transactions". - // which means we probably still only want one writeTx at a time - // even for badger. - writeCount int - writeByteCount int - - o Txo - sn int64 // serial number - - done bool -} - -func (tx *BadgerTx) IsDone() (done bool) { - tx.mu.Lock() - done = tx.done - tx.mu.Unlock() - return -} - -func (tx *BadgerTx) Group() *TxGroup { - return tx.o.Group -} - -func (tx *BadgerTx) Options() Txo { - return tx.o -} - -func (tx *BadgerTx) Type() string { - return BadgerTxn -} - -func (tx *BadgerTx) UseRowCache() bool { - //the row cache speeds up queries. - return rbf.EnableRowCache -} - -// overWriteOurAllocs provides detection of memory -// access outside the transactional context, similar to the -// old school electric fence techniques but without setting -// memory mappings to read-only... instead we just zero -// out the memory allocated to roaring containers by a -// transaction after the commit or rollback. This, -// hopefully, will cause some downstream confusion and -// test failures, which we can use to locate who has been -// holding on to memory they should have copied prior -// to transaction commit. -func (tx *BadgerTx) overWriteOurAllocs() { - - tx.acMu.Lock() - defer tx.acMu.Unlock() - for _, s := range tx.ourAllocs { - - // The Go compiler recognizes the following pattern and inserts - // an efficient memclr instruction. - // See https://github.com/golang/go/issues/5373 - // and https://codereview.appspot.com/137880043 - for i := range s { - s[i] = 0 - // or - // Seebs suggested we might see even more crashes :) - // but since it will be slow (no memclr), we'll leave the default 0 for now. - //s[i] = -2 - } - } - // keep this around if we need to activate out-of-mmap memory access again. - //for _, v := range tx.ourContainers { - //v.Invalid = true - //v.Tx = tx - //} -} - -// Pointer gives us a memory address for the underlying transaction for debugging. -// It is public because we use it in roaring to report invalid container memory access -// outside of a transaction. -func (tx *BadgerTx) Pointer() string { - return fmt.Sprintf("%p", tx) -} - -// Rollback rolls back the transaction. -func (tx *BadgerTx) Rollback() { - tx.mu.Lock() - defer tx.mu.Unlock() - - //pp("BadgerTx.Rollback p=%p, its: '%v' initloc: '%v',\n rollbackloc:'%v'", tx, tx.Db.UnprotectedListOpenItAsString(), tx.initloc, stack()) - tx.tx.Discard() // must hold tx.mu mutex lock - - tx.Db.muOpenTxIt.Lock() - delete(tx.Db.openTx, tx) - tx.Db.muOpenTxIt.Unlock() - - if tx.doAllocZero { - // and clear our allocs, to find code using them outside of a txn. - tx.overWriteOurAllocs() - } - - tx.Db.unprotectedCleanupTx(tx) -} - -// Commit commits the transaction to permanent storage. -// Commits can handle up to 100k updates to fragments -// at once, but not more. This is a BadgerDB imposed limit. -func (tx *BadgerTx) Commit() error { - tx.mu.Lock() - defer tx.mu.Unlock() - - tx.Db.muOpenTxIt.Lock() - delete(tx.Db.openTx, tx) - tx.Db.muOpenTxIt.Unlock() - - //pp("BadgerTx.Commit (write:%v) p=%p, stackID=%x openit: '%v' initloc: '%v', commitloc:\n%v", tx.write, tx, stackID, tx.Db.UnprotectedListOpenItAsString(), tx.initloc, stack()) - - err := tx.tx.CommitAt(1, nil) // must hold tx.mu mutex lock - - if tx.doAllocZero { - tx.overWriteOurAllocs() - } - - tx.Db.unprotectedCleanupTx(tx) - return err -} - -// Readonly returns true iff the BadgerTx is read-only. -func (tx *BadgerTx) Readonly() bool { - return !tx.write -} - -// RoaringBitmap returns the roaring.Bitmap for all bits in the fragment. -func (tx *BadgerTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - - return tx.OffsetRange(index, field, view, shard, 0, 0, LeftShifted16MaxContainerKey) -} - -// Container returns the requested roaring.Container, selected by fragment and ckey -func (tx *BadgerTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) { - - // values returned from Get() are only valid while the transaction - // is open. If you need to use a value outside of the transaction then - // you must use copy() to copy it to another byte slice. - // BUT here we are already inside the Txn. - - bkey := txkey.Key(index, field, view, shard, ckey) - tx.mu.Lock() - var item *badger.Item - item, err = tx.tx.Get(bkey) - tx.mu.Unlock() - if err == badger.ErrKeyNotFound { - // Seems crazy, but we, for now at least, - // match what RoaringTx does by returning nil, nil. - return nil, nil - } else { - panicOn(err) - } - - err = item.Value(func(v []byte) error { - // This func with val would only be called if item.Value encounters no error - c = tx.toContainer(item.UserMeta(), v) - return nil - }) - panicOn(err) - return -} - -func (w *BadgerDBWrapper) NewWriteBatch() { - w.muDb.Lock() - defer w.muDb.Unlock() - if w.writeBatch != nil { - panic("must FlushWriteBatch() before calling NewWriteBatch()") - } - w.writeBatch = w.db.NewWriteBatch() -} - -// Flush any remaining un-committed writes in progress. -func (w *BadgerDBWrapper) FlushWriteBatch() (err error) { - w.muDb.Lock() - defer w.muDb.Unlock() - if w.writeBatch == nil { - panic("CommitWriteBatch error: no batch in progress") - } - err = w.writeBatch.Flush() - w.writeBatch = nil - return -} - -// Cancel any remaining un-committed writes in progress. -func (w *BadgerDBWrapper) CancelWriteBatch() { - w.muDb.Lock() - defer w.muDb.Unlock() - if w.writeBatch == nil { - panic("CancelWriteBatch error: no batch in progress") - } - w.writeBatch.Cancel() -} - -// PutContainer stores rc under the specified fragment and container ckey. -func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey uint64, rc *roaring.Container) error { - - bkey := txkey.Key(index, field, view, shard, ckey) - var by []byte - - ct := roaring.ContainerType(rc) - - switch ct { - case roaring.ContainerArray: - by = fromArray16(roaring.AsArray(rc)) - case roaring.ContainerBitmap: - by = fromArray64(roaring.AsBitmap(rc)) - case roaring.ContainerRun: - by = fromInterval16(roaring.AsRuns(rc)) - case roaring.ContainerNil: - panic("wat? nil roaring.Container is unexpected, no?!?") - default: - panic(fmt.Sprintf("unknown roaring.Container type: %v", ct)) - } - - entry := badger.NewEntry(bkey, by).WithMeta(ct) - - tx.Db.muDb.Lock() - if tx.Db.writeBatch != nil { - err := tx.Db.writeBatch.SetEntry(entry) // Will create txns as needed. - tx.Db.muDb.Unlock() - return err - } - tx.Db.muDb.Unlock() - - tx.mu.Lock() - defer tx.mu.Unlock() - - tx.writeCount++ - sz := len(by) + len(bkey) + 2 - tx.writeByteCount += sz - - // The integration tests do large bit level loads that exceed 10MB. - // So we autocommit and start a new Txn if we are about to - // write too much into one Txn. - // - // The badger defaults limits are currently: - // maxBatchCount:104857, maxBatchSize:10066329 - // - // However, emprirically we still get ErrTnTooBig when - // tx.writeByteCount=5884222; or when tx.writeCount=16197. - // So duck under both those thresholds by some margin. - if tx.writeCount > 100 || tx.writeByteCount > 2000000 { - // avoid ErrTxnTooBig by commiting before going over the limits, - // because then we get a error: "Transaction Conflict. Please retry." - err := tx.tx.CommitAt(1, nil) - panicOn(err) - // badger docs: - // `ErrConflict is returned when a transaction conflicts with another transaction. This can - // happen if the read rows had been updated concurrently by another transaction. - // ErrConflict = errors.New("Transaction Conflict. Please retry")` - //if err == badger.ErrConflict { - // problem is, we don't have the previous entry handy now. - //} - if err != nil { - panic(fmt.Sprintf("commit failed on bkey '%v': err '%v'", string(bkey), err)) - } - tx.tx = tx.Db.db.NewTransactionAt(1, tx.write) - - tx.writeCount = 1 - tx.writeByteCount = sz - } - err := tx.tx.SetEntry(entry) - - // ErrTxnTooBig is returned if too many writes are fit into a single transaction. - // badger docs: "An ErrTxnTooBig will be reported in case the number of pending - // writes/deletes in the transaction exceeds a certain limit. In that case, it - // is best to commit the transaction and start a new transaction immediately." - // - if err == badger.ErrTxnTooBig { - - err = tx.tx.CommitAt(1, nil) - if err != nil { - panic(fmt.Sprintf("commit after TooBig failed on bkey '%v': err '%v'", string(bkey), err)) - } - - tx.tx = tx.Db.db.NewTransactionAt(1, tx.write) - //vv("NewTx write txn (p=%p) on gid=%v. b/c TooBig writeCount=%v; writeByteCount=%v", tx.tx, curGID(), tx.writeCount, tx.writeByteCount) - tx.writeCount = 1 - tx.writeByteCount = sz - - err = tx.tx.SetEntry(entry) - panicOn(err) - //panic(fmt.Sprintf("got error badger.ErrTxnTooBig, but we shoud never get this now; len(by) = %v; len(bkey)=%v; vs limit is 10MB. tx.writeCount=%v; tx.writeByteCount=%v;", len(by), len(bkey), tx.writeCount, tx.writeByteCount)) - } - return err -} - -// RemoveContainer deletes the container specified by the shard and container key ckey -func (tx *BadgerTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error { - bkey := txkey.Key(index, field, view, shard, ckey) - tx.mu.Lock() - err := tx.tx.Delete(bkey) - tx.mu.Unlock() - return err -} - -// Add sets all the a bits hot in the specified fragment. -func (tx *BadgerTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) { - return tx.addOrRemove(index, field, view, shard, batched, false, a...) -} - -// Remove clears all the specified a bits in the chosen fragment. -func (tx *BadgerTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - const batched = false - const remove = true - return tx.addOrRemove(index, field, view, shard, batched, remove, a...) -} - -func (tx *BadgerTx) addOrRemove(index, field, view string, shard uint64, batched, remove bool, a ...uint64) (changeCount int, err error) { - // pure hack to match RoaringTx - defer func() { - if !remove && !batched { - if changeCount > 0 { - changeCount = 1 - } - } - }() - - if len(a) == 0 { - return 0, nil - } - - // have to sort, b/c input is not always sorted. - sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) - - var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. - var rc *roaring.Container - var hi uint64 - var lo uint16 - - for i, v := range a { - - hi, lo = highbits(v), lowbits(v) - if hi != lastHi { - // either first time through, or changed to a different container. - // do we need put the last updated container now? - if i > 0 { - // not first time through, write what we got. - if remove && (rc == nil || rc.N() == 0) { - err = tx.RemoveContainer(index, field, view, shard, lastHi) - panicOn(err) - } else { - err = tx.PutContainer(index, field, view, shard, lastHi, rc) - panicOn(err) - } - } - // get the next container - rc, err = tx.Container(index, field, view, shard, hi) - panicOn(err) - } // else same container, keep adding bits to rct. - chng := false - // rc can be nil before, and nil after, in both Remove/Add below. - // The roaring container add() and remove() methods handle this. - if remove { - rc, chng = rc.Remove(lo) - } else { - rc, chng = rc.Add(lo) - } - if chng { - changeCount++ - } - lastHi = hi - } - // write the last updates. - if remove { - if rc == nil || rc.N() == 0 { - err = tx.RemoveContainer(index, field, view, shard, hi) - panicOn(err) - } else { - err = tx.PutContainer(index, field, view, shard, hi, rc) - panicOn(err) - } - } else { - if rc == nil || rc.N() == 0 { - panic("there should be no way to have an empty bitmap AFTER an Add() operation") - } - err = tx.PutContainer(index, field, view, shard, hi, rc) - panicOn(err) - } - return -} - -// Contains returns exists true iff the bit chosen by key is -// hot (set to 1) in specified fragment. -func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { - - lo, hi := lowbits(key), highbits(key) - bkey := txkey.Key(index, field, view, shard, hi) - tx.mu.Lock() - item, err := tx.tx.Get(bkey) - tx.mu.Unlock() - if err == badger.ErrKeyNotFound { - return false, nil - } - if err != nil { - return false, err - } - err = item.Value(func(v []byte) error { - // This func with val would only be called if item.Value encounters no error - c := tx.toContainer(item.UserMeta(), v) - exists = c.Contains(lo) - return nil - }) - return exists, err -} - -func (tx *BadgerTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { - - prefix := txkey.AllShardPrefix(index, field, view) - - bi := NewBadgerIterator(tx, prefix) - defer bi.Close() - bi.Seek(prefix) - if !bi.it.Valid() { - return - } - lastShard := uint64(0) - firstDone := false - for bi.Next() { - item := bi.it.Item() - key := item.Key() - shard := txkey.ShardFromKey(key) - if firstDone { - if shard != lastShard { - sliceOfShards = append(sliceOfShards, shard) - } - lastShard = shard - } else { - // first time - lastShard = shard - firstDone = true - sliceOfShards = append(sliceOfShards, shard) - } - - } - return -} - -// key is the container key for the first roaring Container -// roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will -// return the first container at or after key. found will be true if a -// container is found at key. -// -// BadgerTx notes: We auto-stop at the end of this shard, not going beyond. -func (tx *BadgerTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { - - // needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000" - needle := txkey.Key(index, field, view, shard, firstRoaringContainerKey) - - // prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@" - prefix := txkey.Prefix(index, field, view, shard) - - bi := NewBadgerIterator(tx, prefix) - bi.Seek(needle) - if !bi.it.Valid() { - return bi, false, nil - } - - if !bi.it.ValidForPrefix(prefix) { - return bi, false, nil - } - item := bi.it.Item() - // have to compare b/c badger might give us valid iterator - // that is past our needle if needle isn't present. - return bi, bytes.Equal(item.Key(), needle), nil -} - -// BadgerIterator is the iterator returned from a BadgerTx.ContainerIterator() call. -// It implements the roaring.ContainerIterator interface. -type BadgerIterator struct { - tx *BadgerTx - it *badger.Iterator - - prefix []byte - seekto []byte - - // seen counts how many Next() calls we have seen. - // It is used to match roaring.ContainerIterator semantics. - // Also useful for testing. - seen int -} - -// NewBadgerIterator creates an iterator on tx that will -// only return txkey.Keys that start with prefix. -func NewBadgerIterator(tx *BadgerTx, prefix []byte) (bi *BadgerIterator) { - - opts := badger.DefaultIteratorOptions - opts.PrefetchValues = false // else by default, pre-fetches the 1st 100 values, which would be slow. - opts.Reverse = false - - tx.mu.Lock() - it := tx.tx.NewIterator(opts) - tx.mu.Unlock() - - bi = &BadgerIterator{ - tx: tx, - it: it, - prefix: prefix, - } - tx.Db.muOpenTxIt.Lock() - if tx.Db.openIt == nil { - tx.Db.openIt = make(map[*BadgerIterator]bool) - } - tx.Db.openIt[bi] = false // true for reverse, false for forward iteration. - tx.Db.muOpenTxIt.Unlock() - - bi.it.Seek(prefix) - return -} - -// NewBadgerReverseIterator makes a highest-to-lowest key iterator. -// Only keys that are prefixed with prefix will be returned. -// seekto tells where to start, and should be typically shard+1 -// to start at the end of shard. Really only used in Max() at the moment. -// After creating a reverse badger iterator it, we will call it.Seek(seekto). -func NewBadgerReverseIterator(tx *BadgerTx, prefix, seekto []byte) (bi *BadgerIterator) { - - tx.Db.muOpenTxIt.Lock() - defer tx.Db.muOpenTxIt.Unlock() - - opts := badger.DefaultIteratorOptions - opts.PrefetchValues = false // else by default, pre-fetches the 1st 100 values, which would be slow. - opts.Reverse = true - opts.Prefix = prefix // possible storage IOPs optimization by badger - tx.mu.Lock() - it := tx.tx.NewIterator(opts) - tx.mu.Unlock() - - bi = &BadgerIterator{ - tx: tx, - it: it, - prefix: prefix, - seekto: seekto, - } - if tx.Db.openIt == nil { - tx.Db.openIt = make(map[*BadgerIterator]bool) - } - bi.tx.Db.openIt[bi] = true // true for reverse, false for forward iteration. - bi.it.Seek(seekto) - return -} - -// Close tells the database and transaction that the user is done -// with the iterator. -// From the badger docs: It is important to call this when you're done with iteration. -// else you will get an error on tx.Discard()/Commit(). -func (bi *BadgerIterator) Close() { - - bi.tx.Db.muOpenTxIt.Lock() - delete(bi.tx.Db.openIt, bi) - bi.it.Close() - - bi.tx.Db.muOpenTxIt.Unlock() -} - -// Valid returns false if there are no more values in the iterator's range. -func (bi *BadgerIterator) Valid() bool { - return bi.it.Valid() -} - -// Seek allows the iterator to start at needle instead of the global begining. -func (bi *BadgerIterator) Seek(needle []byte) { - bi.it.Seek(needle) -} - -// Next advances the iterator. -func (bi *BadgerIterator) Next() bool { - - // have to skip the first bi.it.Next() call because badger iterators point to the - // first value immediately, but Pilosa iterators must have Next() called - // on a fresh iterator to get the first value. - if bi.seen > 0 { - bi.it.Next() - } - bi.seen++ - return bi.it.ValidForPrefix(bi.prefix) // does the bi.it.Valid() inside and false if not valid always. -} - -// Value retrieves what is pointed at currently by the iterator. -func (bi *BadgerIterator) Value() (containerKey uint64, c *roaring.Container) { - if !bi.it.Valid() { - panic("bi.it not valid") - } - item := bi.it.Item() - if item == nil { - panic("item was nil") - } - key := item.Key() - containerKey = txkey.KeyExtractContainerKey(key) - - err := item.Value(func(v []byte) error { - c = bi.tx.toContainer(item.UserMeta(), v) - return nil - }) - panicOn(err) - return -} - -// badgerFinder implements roaring.IteratorFinder. -// It is used by BadgerTx.ForEach() -type badgerFinder struct { - tx *BadgerTx - index string - field string - view string - shard uint64 - needClose []Closer -} - -// FindIterator lets badgerFinder implement the roaring.FindIterator interface. -func (bf *badgerFinder) FindIterator(seek uint64) (roaring.ContainerIterator, bool) { - a, found, err := bf.tx.ContainerIterator(bf.index, bf.field, bf.view, bf.shard, seek) - panicOn(err) - bf.needClose = append(bf.needClose, a) - return a, found -} - -// Close closes all bf.needClose listed Closers. -func (bf *badgerFinder) Close() { - for _, i := range bf.needClose { - i.Close() - } -} - -// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE -// the transaction Commits or Rollsback. -func (tx *BadgerTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { - bf := &badgerFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} - itr := roaring.NewIterator(bf) - return itr -} - -// ForEach applies fn to each bitmap in the fragment. -func (tx *BadgerTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { - itr := tx.NewTxIterator(index, field, view, shard) - defer itr.Close() - - // Seek can create many container iterators, thus bf.Close() needClose list. - itr.Seek(0) - // v is the bit we are operating on. - for v, eof := itr.Next(); !eof; v, eof = itr.Next() { - if err := fn(v); err != nil { - return err - } - } - return nil -} - -// ForEachRange applies fn on the selected range of bits on the chosen fragment. -func (tx *BadgerTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { - - itr := tx.NewTxIterator(index, field, view, shard) - defer itr.Close() - - itr.Seek(start) - - // v is the bit we are operating on. - for v, eof := itr.Next(); !eof && v < end; v, eof = itr.Next() { - if err := fn(v); err != nil { - return err - } - } - return nil -} - -// Count operates on the full bitmap level, so it sums over all the containers -// in the bitmap. -func (tx *BadgerTx) Count(index, field, view string, shard uint64) (uint64, error) { - - a, found, err := tx.ContainerIterator(index, field, view, shard, 0) - panicOn(err) - defer a.Close() - if !found { - return 0, nil - } - result := int32(0) - for a.Next() { - ckey, cont := a.Value() - _ = ckey - result += cont.N() - } - - return uint64(result), nil -} - -// Max is the maximum bit-value in your bitmap. -// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does. -func (tx *BadgerTx) Max(index, field, view string, shard uint64) (uint64, error) { - - prefix := txkey.Prefix(index, field, view, shard) - seekto := txkey.Prefix(index, field, view, shard+1) - - it := NewBadgerReverseIterator(tx, prefix, seekto) // this iterator is still open, when we commit/discard tx. - defer it.Close() - - if !it.it.Valid() { - return 0, nil - } - hb, rc := it.Value() // getting it returns invalid, as in empty iterator - lb := rc.Max() - - return hb<<16 | uint64(lb), nil -} - -// Min returns the smallest bit set in the fragment. If no bit is hot, -// the second return argument is false. -func (tx *BadgerTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { - - // Seek can create many container iterators, thus the bf.Close() needClose list. - bf := &badgerFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} - defer bf.Close() - itr := roaring.NewIterator(bf) - - itr.Seek(0) - - // v is the bit we are operating on. - v, eof := itr.Next() - if eof { - return 0, false, nil - } - return v, true, nil -} - -// UnionInPlace unions all the others Bitmaps into a new Bitmap, and then writes it to the -// specified fragment. -func (tx *BadgerTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - panicOn(err) - - rbm.UnionInPlace(others...) - // iterate over the containers that changed within rbm, and write them back to disk. - - it, found := rbm.Containers.Iterator(0) - _ = found // don't care about the value of found, because first containerKey might be > 0 - - for it.Next() { - containerKey, rc := it.Value() - - // TODO: only write the changed ones back, as optimization? - // Compare to ImportRoaringBits. - err := tx.PutContainer(index, field, view, shard, containerKey, rc) - panicOn(err) - } - return nil -} - -// CountRange returns the count of hot bits in the start, end range on the fragment. -// roaring.countRange counts the number of bits set between [start, end). -func (tx *BadgerTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { - - if tx.frag == nil { - return tx.countRangeNoFrag(index, field, view, shard, start, end) - } - - // For speed, exploit the fact that on startup the rowCache will - // have already loaded fragments. - rowID := start / ShardWidth - row, err := tx.frag.unprotectedRow(tx, rowID) - if err != nil { - return 0, err - } - return row.Count(), nil -} - -// CountRange returns the count of hot bits in the start, end range on the fragment. -// roaring.countRange counts the number of bits set between [start, end). -func (tx *BadgerTx) countRangeNoFrag(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { - - if start >= end { - return 0, nil - } - - skey := highbits(start) - ekey := highbits(end) - - citer, found, err := tx.ContainerIterator(index, field, view, shard, skey) - _ = found - panicOn(err) - - defer citer.Close() - - // If range is entirely in one container then just count that range. - if skey == ekey { - citer.Next() - _, c := citer.Value() - return uint64(c.CountRange(int32(lowbits(start)), int32(lowbits(end)))), nil - } - - for citer.Next() { - k, c := citer.Value() - if k < skey { - citer.Close() - panic(fmt.Sprintf("should be impossible for k(%v) to be less than skey(%v). tx p=%p", k, skey, tx)) - } - - // k > ekey handles the case when start > end and where start and end - // are in different containers. Same container case is already handled above. - if k > ekey { - break - } - if k == skey { - n += uint64(c.CountRange(int32(lowbits(start)), roaring.MaxContainerVal+1)) - continue - } - if k < ekey { - n += uint64(c.N()) - continue - } - if k == ekey { - n += uint64(c.CountRange(0, int32(lowbits(end)))) - break - } - } - - return n, nil -} - -// OffsetRange creates a new roaring.Bitmap to return in other. For all the -// hot bits in [start, endx) of the chosen fragment, it stores -// them into other but with offset added to their bit position. -// The primary client is doing this, using ShardWidth, already; see -// fragment.rowFromStorage() in fragment.go. For example: -// -// data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard, -// f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) -// ^ offset ^ start ^ endx -// -// The start and endx arguments are container keys that have been shifted left by 16 bits; -// their highbits() will be taken to determine the actual container keys. This -// is done to conform to the roaring.OffsetRange() argument convention. -// -func (tx *BadgerTx) OffsetRange(index, field, view string, shard, offset, start, endx uint64) (other *roaring.Bitmap, err error) { - - // roaring does these three checks in its OffsetRange - if lowbits(offset) != 0 { - panic("offset must not contain low bits") - } - if lowbits(start) != 0 { - panic("range start must not contain low bits") - } - if lowbits(endx) != 0 { - panic("range end must not contain low bits") - } - - other = roaring.NewSliceBitmap() - off := highbits(offset) - hi0, hi1 := highbits(start), highbits(endx) - - needle := txkey.Key(index, field, view, shard, hi0) - prefix := txkey.Prefix(index, field, view, shard) - - n2, pre2 := txkey.KeyAndPrefix(index, field, view, shard, hi0) - if string(n2) != string(needle) { - panic(fmt.Sprintf("problem! n2(%v) != needle(%v), txkey.KeyAndPrefix not consitent with txkey.Key()", string(n2), string(needle))) - } - if string(pre2) != string(prefix) { - panic(fmt.Sprintf("problem! pre2(%v) != prefix(%v), txkey.KeyAndPrefix not consitent with txkey.Key()", string(pre2), string(prefix))) - } - - it := NewBadgerIterator(tx, prefix) - defer it.Close() - it.Seek(needle) - for ; it.it.ValidForPrefix(prefix); it.Next() { - item := it.it.Item() - bkey := item.Key() - k := txkey.KeyExtractContainerKey(bkey) - - // >= hi1 is correct b/c endx cannot have any lowbits set. - if uint64(k) >= hi1 { - break - } - destCkey := off + (k - hi0) - err := item.Value(func(v []byte) error { - c := tx.toContainer(item.UserMeta(), v) - other.Containers.Put(destCkey, c.Freeze()) - - return nil - }) - if err != nil { - return nil, err - } - } - return other, nil -} - -// IncrementOpN increments the tx opcount by changedN -func (tx *BadgerTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - tx.opcount += changedN -} - -// ImportRoaringBits handles deletes by setting clear=true. -// rowSet[rowID] returns the number of bit changed on that rowID. -func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { - n := itr.Len() - if n == 0 { - return - } - rowSet = make(map[uint64]int) - - var currRow uint64 - - var oldC *roaring.Container - for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { - if rowSize != 0 { - currRow = itrKey / rowSize - } - nsynth := int(synthC.N()) - if nsynth == 0 { - continue - } - // INVAR: nsynth > 0 - - oldC, err = tx.Container(index, field, view, shard, itrKey) - panicOn(err) - if err != nil { - return - } - - if oldC == nil || oldC.N() == 0 { - // no container at the itrKey in badger (or all zero container). - if clear { - // changed of 0 and empty rowSet is perfect, no need to change the defaults. - continue - } else { - - changed += nsynth - rowSet[currRow] += nsynth - - err = tx.PutContainer(index, field, view, shard, itrKey, synthC) - if err != nil { - return - } - continue - } - } - - if clear { - existN := oldC.N() // number of bits set in the old container - newC := oldC.Difference(synthC) - - // update rowSet and changes - if newC.N() == existN { - // INVAR: do changed need adjusting? nope. same bit count, - // so no change could have happened. - continue - } else { - changes := int(existN - newC.N()) - changed += changes - rowSet[currRow] -= changes - - if tx.DeleteEmptyContainer && newC.N() == 0 { - err = tx.RemoveContainer(index, field, view, shard, itrKey) - if err != nil { - return - } - continue - } - err = tx.PutContainer(index, field, view, shard, itrKey, newC) - if err != nil { - return - } - continue - } - } else { - // setting bits - - existN := oldC.N() - if existN == roaring.MaxContainerVal+1 { - // completely full container already, set will do nothing. so changed of 0 default is perfect. - continue - } - if existN == 0 { - // can nsynth be zero? No, because of the continue/invariant above where nsynth > 0 - changed += nsynth - rowSet[currRow] += nsynth - err = tx.PutContainer(index, field, view, shard, itrKey, synthC) - if err != nil { - return - } - continue - } - - newC := roaring.Union(oldC, synthC) // UnionInPlace was giving us crashes on overly large containers. - - if roaring.ContainerType(newC) == roaring.ContainerBitmap { - newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it. - } - if newC.N() != existN { - changes := int(newC.N() - existN) - changed += changes - rowSet[currRow] += changes - - err = tx.PutContainer(index, field, view, shard, itrKey, newC) - if err != nil { - panicOn(err) - return - } - continue - } - } - } - return -} - -func (tx *BadgerTx) toContainer(typ byte, v []byte) (c *roaring.Container) { - - if len(v) == 0 { - return nil - } - - var w []byte - useRowCache := tx.UseRowCache() - if tx.doAllocZero || useRowCache { - // Do electric fence-inspired bad-memory read detection. - // - // The v []byte lives in BadgerDB's memory-mapped vlog-file, - // and Badger will recycle it after tx ends with rollback or commit. - // - // Problem is, at least some operations were not respecting transaction boundaries. - // This technique helped us find them. The rowCache was an example. - // - // See the global const DetectMemAccessPastTx - // at the top of txfactory.go to activate/deactivate this. - // - // Seebs suggested this nice variation: we could use individual mmaps for these - // copies, which would be unusable in production, but workable for testing, and then unmap them, - // which would get us probable segfaults on future accesses to them. - // - // The go runtime also has an -efence flag which may be similarly useful if really pressed. - // - w = make([]byte, len(v)) - copy(w, v) - - if !useRowCache { - // register w so we can catch out-of-tx memory access - tx.acMu.Lock() - defer tx.acMu.Unlock() - tx.ourAllocs = append(tx.ourAllocs, w) - } - } else { - w = v - } - - switch typ { - case roaring.ContainerArray: - c = roaring.NewContainerArray(toArray16(w)) - if tx.doAllocZero { - // tx.acMu was acquired above, and Unlock deferred. - tx.ourContainers = append(tx.ourContainers, c) - } - case roaring.ContainerBitmap: - c = roaring.NewContainerBitmap(-1, toArray64(w)) - if tx.doAllocZero { - // tx.acMu was acquired above, and Unlock deferred. - tx.ourContainers = append(tx.ourContainers, c) - } - case roaring.ContainerRun: - c = roaring.NewContainerRun(toInterval16(w)) - if tx.doAllocZero { - // tx.acMu was acquired above, and Unlock deferred. - tx.ourContainers = append(tx.ourContainers, c) - } - default: - panic(fmt.Sprintf("unknown container: %v", typ)) - } - c.SetMapped(true) - return c -} - -// StringifiedBadgerKeys returns a string with all the container -// keys available in badger. -func (w *BadgerDBWrapper) StringifiedBadgerKeys(optionalUseThisTx Tx, short bool) (r string) { - if optionalUseThisTx == nil { - tx, _ := w.NewTx(!writable, "", Txo{}) - defer tx.Rollback() - r = stringifiedBadgerKeysTx(tx.(*BadgerTx), short) - return - } - - btx, ok := optionalUseThisTx.(*BadgerTx) - if !ok { - return fmt.Sprintf("", optionalUseThisTx) - } - r = stringifiedBadgerKeysTx(btx, short) - return -} - -// countBitsSet returns the number of bits set (or "hot") in -// the roaring container value found by the txkey.Key() -// formatted bkey. -func (tx *BadgerTx) countBitsSet(bkey []byte) (n int) { - - item, err := tx.tx.Get(bkey) - if err == badger.ErrKeyNotFound { - panic(fmt.Sprintf("badger did not have value for bkey = '%v'", string(bkey))) - } - panicOn(err) - - var rc *roaring.Container - err = item.Value(func(v []byte) error { - // This func with val would only be called if item.Value encounters no error - rc = tx.toContainer(item.UserMeta(), v) - return nil - }) - panicOn(err) - - n = int(rc.N()) - return -} - -func (tx *BadgerTx) Dump(short bool, shard uint64) { - fmt.Printf("BadgerTx %p Dump: %v\n", tx, stringifiedBadgerKeysTx(tx, short)) -} - -// stringifiedBadgerKeysTx reports all the badger keys and a -// corresponding blake3 hash viewable by txn within the entire -// badger database. -// It also reports how many bits are hot in the roaring container -// (how many bits are set, or 1 rather than 0). -// -// By convention, we must return the empty string if there -// are no keys present. The tests use this to confirm -// an empty database. -func stringifiedBadgerKeysTx(tx *BadgerTx, short bool) (r string) { - - r = "allkeys:[\n" - it := tx.tx.NewIterator(badger.DefaultIteratorOptions) // PrefetchValues true okay here. - defer it.Close() - any := false - for it.Rewind(); it.Valid(); it.Next() { - any = true - item := it.Item() - bkey := item.Key() - key := txkey.ToString(bkey) - ckey := txkey.KeyExtractContainerKey(bkey) - h := "" - srbm := "" - err := item.Value(func(val []byte) error { - h = hash.Blake3sum16(val) - ct := tx.toContainer(item.UserMeta(), val) - cts := roaring.NewSliceContainers() - cts.Put(ckey, ct) - rbm := &roaring.Bitmap{Containers: cts} - srbm = BitmapAsString(rbm) - return nil - }) - panicOn(err) - r += fmt.Sprintf("%v -> %v (%v hot)\n", key, h, tx.countBitsSet(bkey)) - if !short { - r += " ......." + srbm + "\n" - } - } - r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) - - if !any { - return "" - } - return "badger-" + r -} - -func (w *BadgerDBWrapper) DeleteField(index, field, fieldPath string) error { - - // under blue-green roaring_badger, the directory will not be found, b/c roaring will have - // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: - // "If the path does not exist, RemoveAll returns nil (no error)" - err := os.RemoveAll(fieldPath) - if err != nil { - return errors.Wrap(err, "removing directory") - } - prefix := txkey.FieldPrefix(index, field) - return w.DeletePrefix(prefix) -} - -func (w *BadgerDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { - prefix := txkey.Prefix(index, field, view, shard) - return w.DeletePrefix(prefix) -} - -func (w *BadgerDBWrapper) DeletePrefix(prefix []byte) error { - w.muDb.Lock() - defer w.muDb.Unlock() - - // a) do key-ony iteration, no value fetch; - // - // b) do deletes in large batches, to avoid alot of txn overhead; - // per recommendation https://github.com/dgraph-io/badger/issues/598 - // - // c) we do not, at present, try to maintain one large - // transaction with all the keys in a index in it. Because - // there can be too many keys. Hence the index will disappear - // in chucks of 100K keys, not atomically-all-at-once. - - noMoreKeysWithPrefix := false - const maxDeletesPerTxn = 100000 - - for !noMoreKeysWithPrefix { - - txn := w.db.NewTransactionAt(1, true) - //defer txn.Discard() - - o := badger.DefaultIteratorOptions - o.AllVersions = false - o.PrefetchValues = false // key-only iteration, no values. - - // note: panic: Unclosed iterator at time of Txn.Discard ? panic on segfault here? - // This means we messed up and Closed() the Database already; too early. For - // example in TxFactor.CloseIndex() in txfactory.go:331. - it := txn.NewIterator(o) - - //defer it.Close() - n := 0 - goners := make([][]byte, 0, maxDeletesPerTxn) - for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { - - // KeyCopy() is required; Key() means corruption and possible segfault. - key := it.Item().KeyCopy(nil) - goners = append(goners, key) - n++ - if n >= maxDeletesPerTxn { - break - } - } - if !it.ValidForPrefix(prefix) { - noMoreKeysWithPrefix = true // done with the full delete of up to maxDeletesPerTxn - } - for _, key := range goners { - if err := txn.Delete(key); err != nil { - it.Close() - txn.Discard() - return err - } - } - it.Close() - err := txn.CommitAt(1, nil) - - // err back from Update can be ErrConflict in case of - // a conflict. Badger docs: "Depending on the state - // of your application, you have the option to - // retry the operation if you receive this error." - panicOn(err) - - } // end for: proceed to next bath of 100K keys - - // Finally, run a garbage collection to delete values from the value log. - // - // "Only one GC is allowed at a time. If another value log GC - // is running, or DB has been closed, this would return an ErrRejected." - // -- https://godoc.org/github.com/dgraph-io/badger#DB.RunValueLogGC - // Still, we don't see a mutex inside the RunValueLogGC code, so - // lock muGC just to be sure. - w.muGC.Lock() - defer w.muGC.Unlock() - _ = w.db.RunValueLogGC(0.5) - - return nil -} - -func (tx *BadgerTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") - } - var buf bytes.Buffer - sz, err = rbm.WriteTo(&buf) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") - } - return ioutil.NopCloser(&buf), sz, err -} - -func (w *BadgerDBWrapper) OpenListString() (r string) { - return "BadgerDBWrapper.OpenListString not yet implemented" -} -func (w *BadgerDBWrapper) OpenSnList() (sns []int64) { - return nil // panic("BadgerDBWrapper.OpenSnList not yet implemented") -} - -// Sn retreives the serial number of the Tx. -func (tx *BadgerTx) Sn() int64 { - return tx.sn -} - -func (w *BadgerDBWrapper) Update(fn func(txn *badger.Txn) error) error { - txn := w.db.NewTransactionAt(1, true) - defer txn.Discard() - - if err := fn(txn); err != nil { - return err - } - - return txn.CommitAt(1, nil) -} diff --git a/bolt.go b/bolt.go new file mode 100644 index 000000000..9db81fdca --- /dev/null +++ b/bolt.go @@ -0,0 +1,1564 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "math" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/pilosa/pilosa/v2/hash" + "github.com/pilosa/pilosa/v2/rbf" + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/txkey" + "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" +) + +// boltRegistrar facilitates shutdown +// of all the bolt databases started under +// tests. Its needed because most tests don't cleanup +// the *Index(es) they create. But we still +// want to shutdown boltDB goroutines +// after tests run. +// +// It also allows opening the same path twice to +// result in sharing the same open database handle, and +// thus the same transactional guarantees. +// +type boltRegistrar struct { + mu sync.Mutex + mp map[*BoltWrapper]bool + + path2db map[string]*BoltWrapper +} + +func (r *boltRegistrar) Size() int { + r.mu.Lock() + defer r.mu.Unlock() + nmp := len(r.mp) + npa := len(r.path2db) + if nmp != npa { + panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa)) + } + return nmp +} + +var globalBoltReg *boltRegistrar = newBoltTestRegistrar() + +var globalNextTxSnBolt int64 + +func newBoltTestRegistrar() *boltRegistrar { + + return &boltRegistrar{ + mp: make(map[*BoltWrapper]bool), + path2db: make(map[string]*BoltWrapper), + } +} + +// register each bolt created under tests, so we +// can clean them up. This is called by openBoltWrapper() while +// holding the r.mu.Lock, since it needs to atomically +// check the registry and make a new instance only +// if one does not exist for its path, and otherwise +// return the existing instance. +func (r *boltRegistrar) unprotectedRegister(w *BoltWrapper) { + r.mp[w] = true + r.path2db[w.path] = w +} + +// unregister removes w from r +func (r *boltRegistrar) unregister(w *BoltWrapper) { + r.mu.Lock() + delete(r.mp, w) + delete(r.path2db, w.path) + r.mu.Unlock() +} + +func DumpAllBolt() { + short := true + globalBoltReg.mu.Lock() + defer globalBoltReg.mu.Unlock() + for w := range globalBoltReg.mp { + AlwaysPrintf("this bolt path='%v' has: \n%v\n", w.path, w.StringifiedBoltKeys(nil, short)) + } +} + +// boltPath is a helper for determining the full directory +// in which the bolt database will be stored. +func boltPath(path string) string { + if !strings.HasSuffix(path, "-boltdb@") { + return path + "-boltdb@" + } + return path +} + +// openBoltDB opens the database in the bpath directoy +// without deleting any prior content. Any BoltDB +// database directory will have the "-bolt" suffix. +// +// openBoltDB will check the registry and make a new instance only +// if one does not exist for its bpath. Otherwise it returns +// the existing instance. This insures only one boltDB +// per bpath in this pilosa node. +func (r *boltRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper, error) { + path := boltPath(path0) + + r.mu.Lock() + defer r.mu.Unlock() + w, ok := r.path2db[path] + if ok { + // creates the effect of having only one bolt open per pilosa node. + return w, nil + } + // otherwise, make a new bolt and store it in globalBoltReg + + dir := filepath.Dir(path) + if !DirExists(path) { + panicOn(os.MkdirAll(dir, 0755)) + } + + db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize}) + if err != nil { + return nil, errors.Wrapf(err, fmt.Sprintf("open bolt path '%v'", path)) + } + + err = db.Update(func(tx *bolt.Tx) (err error) { + _, err = tx.CreateBucketIfNotExists(bucketCT) + return + }) + if err != nil { + return nil, errors.Wrapf(err, fmt.Sprintf("create bolt bucket '%v' in path '%v'", string(bucketCT), path)) + } + + name := filepath.Base(path) + w = &BoltWrapper{ + name: name, + db: db, + reg: r, + path: path, + doAllocZero: doAllocZero, + openTx: make(map[*BoltTx]bool), + } + r.unprotectedRegister(w) + + return w, nil +} + +func (w *BoltWrapper) Path() string { + return w.path +} + +func (w *BoltWrapper) HasData() (has bool, err error) { + + tx, err := w.NewTx(!writable, "", Txo{}) + if err != nil { + return false, errors.Wrap(err, "HasData NewTx") + } + defer tx.Rollback() + + bi := NewBoltIterator(tx.(*BoltTx), nil) + defer bi.Close() + + for bi.Next() { + return true, nil + } + return false, nil +} + +func (w *BoltWrapper) CleanupTx(tx Tx) { + // inlined into Rollback and Commit, so this is a no-op, just here to satisfy the interface. +} + +func (tx *BoltTx) IsDone() (done bool) { + return atomic.LoadInt64(&tx.unlocked) == 1 +} + +func (w *BoltWrapper) OpenListString() (r string) { + + list := w.listopen() + if len(list) == 0 { + return "" + } + for i, ltx := range list { + if ltx.o.Write { + r += fmt.Sprintf("[%v]write: _sn_ %v %v, \n", i, ltx.sn, ltx.o) + } else { + r += fmt.Sprintf("[%v]read : _sn_ %v %v, \n", i, ltx.sn, ltx.o) + } + } + return +} + +func (w *BoltWrapper) listopen() (slc []*BoltTx) { + w.muDb.Lock() + for v := range w.openTx { + slc = append(slc, v) + } + w.muDb.Unlock() + return +} + +func (w *BoltWrapper) OpenSnList() (slc []int64) { + w.muDb.Lock() + for v := range w.openTx { + slc = append(slc, v.sn) + } + w.muDb.Unlock() + return +} + +// DeleteIndex deletes all the containers associated with +// the named index from the bolt database. +func (w *BoltWrapper) DeleteIndex(indexName string) error { + + // We use the apostrophie rune `'` to locate the end of the + // index name in the key prefix, so we cannot allow indexNames + // themselves to contain apostrophies. + if strings.Contains(indexName, "/") { + return fmt.Errorf("error: bad indexName `%v` in BoltWrapper.DeleteIndex() call: indexName cannot contain '/'.", indexName) + } + prefix := txkey.IndexOnlyPrefix(indexName) + return w.DeletePrefix(prefix) +} + +// statically confirm that BoltTx satisfies the Tx interface. +var _ Tx = (*BoltTx)(nil) + +// BoltWrapper provides the NewTx() method. +type BoltWrapper struct { + db *bolt.DB + + muDb sync.Mutex + + path string + name string + + // track our registrar for Close / goro leak reporting purposes. + reg *boltRegistrar + + // make BoltWrapper.Close() idempotent, avoiding panic on double Close() + closed bool + + // doAllocZero sets the corresponding flag on all new BoltTx. + // When doAllocZero is true, we zero out any data from bolt + // after transcation commit and rollback. This simulates + // what would happen if we were to use the mmap-ed data + // from bolt directly. Currently we copy by default for + // safety because otherwise TestAPI_ImportColumnAttrs sees + // corrupted data. + doAllocZero bool + + DeleteEmptyContainer bool + + openTx map[*BoltTx]bool +} + +func (w *BoltWrapper) SetHolder(h *Holder) { + // don't need it at the moment + //w.h = h +} + +// NewTxWRITE lets us see in the callstack dumps where the WRITE tx are. +// Can't have more than one active write per database, so the +// 2nd one will block until the first finishes. +func (w *BoltWrapper) NewTxWRITE() (*bolt.Tx, error) { + boltTxn, err := w.db.Begin(true) + if err != nil { + if w.db == nil || w.IsClosed() { + return nil, fmt.Errorf("cannot call NewTxWRITE() on closed Bolt database: '%v'", err) + } + return nil, err + } + return boltTxn, nil +} + +// NewTxREAD lets us see in the callstack dumps where the READ tx are. +func (w *BoltWrapper) NewTxREAD() (*bolt.Tx, error) { + boltTxn, err := w.db.Begin(false) + if err != nil { + if w.db == nil || w.IsClosed() { + return nil, fmt.Errorf("cannot call NewTxREAD() on closed Bolt database: '%v'", err) + } + return nil, err + } + return boltTxn, nil +} + +// NewTx produces Bolt based transactions. If +// the transaction will modify data, then the write flag must be true. +// Read-only queries should set write to false, to allow more concurrency. +// Methods on a BoltTx are thread-safe, and can be called from +// different goroutines. +// +// initialIndexName is optional. It is set by the TxFactory from the Txo +// options provided at the Tx creation point. It allows us to recognize +// and isolate cross-index queries more quickly. It can always be empty "" +// but when set is highly useful for debugging. It has no impact +// on transaction behavior. +// +func (w *BoltWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { + + sn := atomic.AddInt64(&globalNextTxSnBolt, 1) + + ////vv("bolt new tx _sn_ %v; openTx='%v', stack \n%v", sn, w.OpenListString(), stack()) + ////vv("bolt new (write=%v, shard=%v) tx _sn_ %v; openTx='%v'", write, o.Shard, sn, w.OpenListString()) + + var boltTxn *bolt.Tx + if write { + // see the WRITE tx on the callstack. + boltTxn, err = w.NewTxWRITE() + if err != nil { + return nil, err + } + } else { + // see the READ tx on the callstack. + boltTxn, err = w.NewTxREAD() + if err != nil { + return nil, err + } + } + + ltx := &BoltTx{ + sn: sn, + write: write, + tx: boltTxn, + Db: w, + frag: o.Fragment, + doAllocZero: w.doAllocZero, + initialIndexName: initialIndexName, + DeleteEmptyContainer: w.DeleteEmptyContainer, + o: o, + gid: curGID(), + } + tx = ltx + + if isDebugRun { + w.muDb.Lock() + w.openTx[ltx] = true + w.muDb.Unlock() + } + return +} + +// Close shuts down the Bolt database. +func (w *BoltWrapper) Close() (err error) { + w.muDb.Lock() + defer w.muDb.Unlock() + if !w.closed { + if isDebugRun { + // complain if there are still Tx in flight, b/c otherwise we will see + // the somewhat mysterious 'panic: should not be in ReadSlot.free() with slot still owned by gid=107043; refCount=1' + if len(w.openTx) > 0 { + AlwaysPrintf("error: cannot close BoltWrapper with Tx still in flight.") + return + } + } + w.reg.unregister(w) + w.closed = true + return w.db.Close() + } + return nil +} + +func (w *BoltWrapper) IsClosed() (closed bool) { + w.muDb.Lock() + closed = w.closed + w.muDb.Unlock() + return +} + +// BoltTx wraps a bolt.Tx and provides the Tx interface +// method implementations. +// The methods on BoltTx are thread-safe, and can be called +// from different goroutines. +type BoltTx struct { + + // mu serializes bolt operations on this single txn instance. + mu sync.Mutex + sn int64 // serial number + + write bool + Db *BoltWrapper + tx *bolt.Tx + frag *fragment + + opcount int + + //initloc string // stack trace of where we were initially created. + + doAllocZero bool + + initialIndexName string + + DeleteEmptyContainer bool + + unlocked int64 + + o Txo + + // NewTx, write operations, Commit and/or Rollback must all take place on + // the same gid and it must the runtime.LockOSThreaded first. Verify + // that we are using the right goroutine in a debug build using the + // gid, stored here, used for NewTx(). + gid uint64 +} + +// sanity check that database is open. +func (tx *BoltTx) sanity() { + if tx.Db.IsClosed() { + panic("cannot operate on closed Bolt") + } +} + +func (tx *BoltTx) Group() *TxGroup { + return tx.o.Group +} + +func (tx *BoltTx) Type() string { + return BoltTxn +} + +func (tx *BoltTx) UseRowCache() bool { + return rbf.EnableRowCache +} + +// Pointer gives us a memory address for the underlying transaction for debugging. +// It is public because we use it in roaring to report invalid container memory access +// outside of a transaction. +func (tx *BoltTx) Pointer() string { + return fmt.Sprintf("%p", tx) +} + +// Rollback rolls back the transaction. +func (tx *BoltTx) Rollback() { + notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) + if !notDone { + return + } + ////vv("bolt rollback tx _sn_ %v; stack \n%v", tx.sn) // , stack()) + if isDebugRun { + tx.sanity() + tx.Db.muDb.Lock() + delete(tx.Db.openTx, tx) + tx.Db.muDb.Unlock() + } + + tx.mu.Lock() + defer tx.mu.Unlock() + + //tx.debugOnlyGidcheck() + _ = tx.tx.Rollback() // must hold tx.mu mutex lock + + tx.o.dbs.Cleanup(tx) +} + +// Commit commits the transaction to permanent storage. +// Commits can handle up to 100k updates to fragments +// at once, but not more. This is a BoltDB imposed limit. +func (tx *BoltTx) Commit() error { + notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) + if !notDone { + ////vv("Commit already done") + return nil + } + ////vv("bolt commit tx _sn_ %v; path = '%v'; stack \n%v", tx.sn, tx.Db.path, stack()) + //DumpAllBolt() + + if isDebugRun { + tx.sanity() + tx.Db.muDb.Lock() + delete(tx.Db.openTx, tx) + tx.Db.muDb.Unlock() + } + tx.mu.Lock() + defer tx.mu.Unlock() + + err := tx.tx.Commit() + panicOn(err) + + tx.o.dbs.Cleanup(tx) + return err +} + +// Readonly returns true iff the BoltTx is read-only. +func (tx *BoltTx) Readonly() bool { + return !tx.write +} + +// RoaringBitmap returns the roaring.Bitmap for all bits in the fragment. +func (tx *BoltTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { + + return tx.OffsetRange(index, field, view, shard, 0, 0, LeftShifted16MaxContainerKey) +} + +// Container returns the requested roaring.Container, selected by fragment and ckey +func (tx *BoltTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) { + + // values returned from Get() are only valid while the transaction + // is open. If you need to use a value outside of the transaction then + // you must use copy() to copy it to another byte slice. + // BUT here we are already inside the Txn. + + bkey := txkey.Key(index, field, view, shard, ckey) + tx.mu.Lock() + + //tx.debugOnlyGidcheck() + + bkt := tx.tx.Bucket(bucketCT) + v := bkt.Get(bkey) + tx.mu.Unlock() + + if v == nil { + // not found + return nil, nil + } + n := len(v) + if n > 0 { + c = tx.toContainer(v[n-1], v[0:(n-1)]) + } + return +} + +var bucketCT = []byte("ct") + +// PutContainer stores rc under the specified fragment and container ckey. +func (tx *BoltTx) PutContainer(index, field, view string, shard uint64, ckey uint64, rc *roaring.Container) error { + + bkey := txkey.Key(index, field, view, shard, ckey) + var by []byte + + ct := roaring.ContainerType(rc) + + switch ct { + case roaring.ContainerArray: + by = fromArray16(roaring.AsArray(rc)) + case roaring.ContainerBitmap: + by = fromArray64(roaring.AsBitmap(rc)) + case roaring.ContainerRun: + by = fromInterval16(roaring.AsRuns(rc)) + case roaring.ContainerNil: + panic("wat? nil container is unexpected, no?!?") + default: + panic(fmt.Sprintf("unknown container type: %v", ct)) + } + tx.mu.Lock() + + bkt := tx.tx.Bucket(bucketCT) + err := bkt.Put(bkey, append(by, ct)) + ////vv("err on put bkey = '%v' was %v", string(bkey), err) + tx.mu.Unlock() + + return err +} + +// RemoveContainer deletes the container specified by the shard and container key ckey +func (tx *BoltTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error { + bkey := txkey.Key(index, field, view, shard, ckey) + tx.mu.Lock() + //tx.debugOnlyGidcheck() + + bkt := tx.tx.Bucket(bucketCT) + err := bkt.Delete(bkey) + + tx.mu.Unlock() + return err +} + +// Add sets all the a bits hot in the specified fragment. +func (tx *BoltTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) { + return tx.addOrRemove(index, field, view, shard, batched, false, a...) +} + +// Remove clears all the specified a bits in the chosen fragment. +func (tx *BoltTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { + const batched = false + const remove = true + return tx.addOrRemove(index, field, view, shard, batched, remove, a...) +} + +func (tx *BoltTx) addOrRemove(index, field, view string, shard uint64, batched, remove bool, a ...uint64) (changeCount int, err error) { + // pure hack to match RoaringTx + defer func() { + if !remove && !batched { + if changeCount > 0 { + changeCount = 1 + } + } + }() + + if len(a) == 0 { + return 0, nil + } + + // have to sort, b/c input is not always sorted. + sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) + + var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. + var rc *roaring.Container + var hi uint64 + var lo uint16 + + for i, v := range a { + + hi, lo = highbits(v), lowbits(v) + if hi != lastHi { + // either first time through, or changed to a different container. + // do we need put the last updated container now? + if i > 0 { + // not first time through, write what we got. + if remove && (rc == nil || rc.N() == 0) { + err = tx.RemoveContainer(index, field, view, shard, lastHi) + panicOn(err) + } else { + err = tx.PutContainer(index, field, view, shard, lastHi, rc) + panicOn(err) + } + } + // get the next container + rc, err = tx.Container(index, field, view, shard, hi) + panicOn(err) + } // else same container, keep adding bits to rct. + chng := false + // rc can be nil before, and nil after, in both Remove/Add below. + // The roaring container add() and remove() methods handle this. + if remove { + rc, chng = rc.Remove(lo) + } else { + rc, chng = rc.Add(lo) + } + if chng { + changeCount++ + } + lastHi = hi + } + // write the last updates. + if remove { + if rc == nil || rc.N() == 0 { + err = tx.RemoveContainer(index, field, view, shard, hi) + panicOn(err) + } else { + err = tx.PutContainer(index, field, view, shard, hi, rc) + panicOn(err) + } + } else { + if rc == nil || rc.N() == 0 { + panic("there should be no way to have an empty bitmap AFTER an Add() operation") + } + err = tx.PutContainer(index, field, view, shard, hi, rc) + panicOn(err) + } + return +} + +// Contains returns exists true iff the bit chosen by key is +// hot (set to 1) in specified fragment. +func (tx *BoltTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { + + lo, hi := lowbits(key), highbits(key) + bkey := txkey.Key(index, field, view, shard, hi) + tx.mu.Lock() + + bkt := tx.tx.Bucket(bucketCT) + v := bkt.Get(bkey) + + tx.mu.Unlock() + if v == nil { + return false, nil + } + n := len(v) + if n > 0 { + c := tx.toContainer(v[n-1], v[0:(n-1)]) + exists = c.Contains(lo) + } + return exists, err +} + +func (tx *BoltTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { + + prefix := txkey.AllShardPrefix(index, field, view) + + bi := NewBoltIterator(tx, prefix) + defer bi.Close() + + lastShard := uint64(0) + firstDone := false + for bi.Next() { + shard := txkey.ShardFromKey(bi.lastKey) + if firstDone { + if shard != lastShard { + sliceOfShards = append(sliceOfShards, shard) + } + lastShard = shard + } else { + // first time + lastShard = shard + firstDone = true + sliceOfShards = append(sliceOfShards, shard) + } + + } + return +} + +// key is the container key for the first roaring Container +// roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will +// return the first container at or after key. found will be true if a +// container is found at key. +// +// BoltTx notes: We auto-stop at the end of this shard, not going beyond. +func (tx *BoltTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { + + // needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000" + needle := txkey.Key(index, field, view, shard, firstRoaringContainerKey) + + // prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@" + prefix := txkey.Prefix(index, field, view, shard) + + bi := NewBoltIterator(tx, prefix) + ok := bi.Seek(needle) + if !ok { + return bi, false, nil + } + + // have to compare b/c bolt might give us valid iterator + // that is past our needle if needle isn't present. + return bi, bytes.Equal(bi.lastKey, needle), nil +} + +// BoltIterator is the iterator returned from a BoltTx.ContainerIterator() call. +// It implements the roaring.ContainerIterator interface. +type BoltIterator struct { + tx *BoltTx + cur *bolt.Cursor + + prefix []byte + seekto []byte + + // seen counts how many Next() calls we have seen. + // It is used to match roaring.ContainerIterator semantics. + // Also useful for testing. + seen int + + lastKey []byte + lastVal []byte // *roaring.Container + lastOK bool + lastConsumed bool +} + +// NewBoltIterator creates an iterator on tx that will +// only return boltKeys that start with prefix. +func NewBoltIterator(tx *BoltTx, prefix []byte) (bi *BoltIterator) { + + tx.mu.Lock() + + bkt := tx.tx.Bucket(bucketCT) + cur := bkt.Cursor() + tx.mu.Unlock() + + bi = &BoltIterator{ + tx: tx, + cur: cur, + prefix: prefix, + } + + return +} + +// Close tells the database and transaction that the user is done +// with the iterator. +func (bi *BoltIterator) Close() { + // no-op +} + +// Valid returns false if there are no more values in the iterator's range. +func (bi *BoltIterator) Valid() bool { + return bi.lastOK +} + +// Seek allows the iterator to start at needle instead of the global begining. +func (bi *BoltIterator) Seek(needle []byte) (ok bool) { + bi.tx.mu.Lock() + defer bi.tx.mu.Unlock() + + bi.seen++ // if ommited, red TestBolt_ContainerIterator_empty_iteration_loop() + + k, v := bi.cur.Seek(needle) + ////vv("seek to needle '%v' gives key '%v'", string(needle), txkey.ToString(k)) + + if len(k) == 0 { + // not found, no keys after needle. + bi.lastKey = nil + bi.lastVal = nil + bi.lastOK = false + bi.lastConsumed = false + return false + } + if len(bi.prefix) > 0 { + ok = bytes.HasPrefix(k, bi.prefix) + if !ok { + bi.lastKey = nil + bi.lastVal = nil + bi.lastOK = false + bi.lastConsumed = false + return false + } + } + + bi.lastKey = k + bi.lastVal = v + bi.lastOK = true + bi.lastConsumed = false + + return true +} + +func (bi *BoltIterator) ValidForPrefix(prefix []byte) bool { + if !bi.lastOK { + return false + } + if len(bi.prefix) == 0 { + return true + } + return bytes.HasPrefix(bi.lastKey, bi.prefix) +} + +func (bi *BoltIterator) String() (r string) { + return fmt.Sprintf("BoltIterator{prefix: '%v', seekto: '%v', seen:%v, lastKey:'%v', lastOK:%v, lastConsumed:%v}", string(bi.prefix), string(bi.seekto), bi.seen, string(bi.lastKey), bi.lastOK, bi.lastConsumed) +} + +// Next advances the iterator. +func (bi *BoltIterator) Next() (ok bool) { + //vv("Next; bi.prefix='%v'", string(bi.prefix)) + if bi.lastOK && !bi.lastConsumed { + //vv("lastOk and not consumed, returning wo doing anything") + bi.seen++ + bi.lastConsumed = true + if len(bi.lastVal) == 0 { + panic("bi.lastVal should not have len 0 if lastOK true") + } + return true + } + + var k, v []byte + if bi.seen == 0 { + if len(bi.prefix) == 0 { + bi.tx.mu.Lock() + k, v = bi.cur.First() + bi.tx.mu.Unlock() + } else { + found := bi.Seek(bi.prefix) + // increments bi.seen for us. + if !found { + bi.lastKey = nil + bi.lastVal = nil + bi.lastOK = false + bi.lastConsumed = false + return false + } + // ready to go + return true + } + } + + bi.seen++ +skipEmpty: + if bi.seen > 1 { + bi.tx.mu.Lock() + k, v = bi.cur.Next() + bi.tx.mu.Unlock() + } + + if len(k) == 0 { + // no more + bi.lastKey = nil + bi.lastVal = nil + bi.lastOK = false + bi.lastConsumed = false + return false + } + if len(bi.prefix) > 0 { + ok = bytes.HasPrefix(k, bi.prefix) + if !ok { + bi.lastKey = nil + bi.lastVal = nil + bi.lastOK = false + bi.lastConsumed = false + return false + } + } + bi.lastKey = k + bi.lastVal = v + if len(v) == 0 { + // actually under !tx.DeleteEmptyContainer, we can have empty containers! + goto skipEmpty + } + bi.lastOK = true + bi.lastConsumed = true + + return true +} + +// Value retrieves what is pointed at currently by the iterator. +func (bi *BoltIterator) Value() (containerKey uint64, c *roaring.Container) { + if !bi.lastOK { + panic("bi.cur not valid") + } + containerKey = txkey.KeyExtractContainerKey(bi.lastKey) + + v := bi.lastVal + n := len(v) + if n > 0 { + c = bi.tx.toContainer(v[n-1], v[0:(n-1)]) + } else { + panic("v should not be empty!") + } + return +} + +// boltFinder implements roaring.IteratorFinder. +// It is used by BoltTx.ForEach() +type boltFinder struct { + tx *BoltTx + index string + field string + view string + shard uint64 + needClose []Closer +} + +// FindIterator lets boltFinder implement the roaring.FindIterator interface. +func (bf *boltFinder) FindIterator(seek uint64) (roaring.ContainerIterator, bool) { + a, found, err := bf.tx.ContainerIterator(bf.index, bf.field, bf.view, bf.shard, seek) + panicOn(err) + bf.needClose = append(bf.needClose, a) + return a, found +} + +// Close closes all bf.needClose listed Closers. +func (bf *boltFinder) Close() { + for _, i := range bf.needClose { + i.Close() + } +} + +// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE +// the transaction Commits or Rollsback. +func (tx *BoltTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { + + bf := &boltFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} + itr := roaring.NewIterator(bf) + return itr +} + +// ForEach applies fn to each bitmap in the fragment. +func (tx *BoltTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { + + itr := tx.NewTxIterator(index, field, view, shard) + defer itr.Close() + + // Seek can create many container iterators, thus bf.Close() needClose list. + itr.Seek(0) + // v is the bit we are operating on. + for v, eof := itr.Next(); !eof; v, eof = itr.Next() { + if err := fn(v); err != nil { + return err + } + } + return nil +} + +// ForEachRange applies fn on the selected range of bits on the chosen fragment. +func (tx *BoltTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { + + itr := tx.NewTxIterator(index, field, view, shard) + defer itr.Close() + + itr.Seek(start) + + // v is the bit we are operating on. + for v, eof := itr.Next(); !eof && v < end; v, eof = itr.Next() { + if err := fn(v); err != nil { + return err + } + } + return nil +} + +// Count operates on the full bitmap level, so it sums over all the containers +// in the bitmap. +func (tx *BoltTx) Count(index, field, view string, shard uint64) (uint64, error) { + + a, found, err := tx.ContainerIterator(index, field, view, shard, 0) + panicOn(err) + defer a.Close() + if !found { + return 0, nil + } + result := int32(0) + for a.Next() { + ckey, cont := a.Value() + _ = ckey + result += cont.N() + } + return uint64(result), nil +} + +// Max is the maximum bit-value in your bitmap. +// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does. +func (tx *BoltTx) Max(index, field, view string, shard uint64) (uint64, error) { + + prefix := txkey.Prefix(index, field, view, shard) + seekto := txkey.Prefix(index, field, view, shard+1) + + bkt := tx.tx.Bucket(bucketCT) + cur := bkt.Cursor() + + var k, v []byte + k, _ = cur.Seek(seekto) + if k == nil { + // we have nothing >= seekto, but we might have stuff before it, and we'll wrap backwards. + k, v = cur.Prev() + if k == nil { + // empty database + return 0, nil + } + } else { + // we found something >= seekto, so backup by 1. + k, v = cur.Prev() + if k == nil { + // nothing before seekto + return 0, nil + } + } + + // have something, are we in [prefix, seekto) ? + cmp := bytes.Compare(k, prefix) + if cmp >= 0 { + // good, got max in k, v + } else { + return 0, nil // nothing in [prefix, seekto). + } + + n := len(v) + if n == 0 { + return 0, nil + } + + hb := txkey.KeyExtractContainerKey(k) + rc := tx.toContainer(v[n-1], v[0:(n-1)]) + + lb := rc.Max() + return hb<<16 | uint64(lb), nil +} + +// Min returns the smallest bit set in the fragment. If no bit is hot, +// the second return argument is false. +func (tx *BoltTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { + + // Seek can create many container iterators, thus the bf.Close() needClose list. + bf := &boltFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} + defer bf.Close() + itr := roaring.NewIterator(bf) + + itr.Seek(0) + + // v is the bit we are operating on. + v, eof := itr.Next() + if eof { + return 0, false, nil + } + return v, true, nil +} + +// UnionInPlace unions all the others Bitmaps into a new Bitmap, and then writes it to the +// specified fragment. +func (tx *BoltTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { + + rbm, err := tx.RoaringBitmap(index, field, view, shard) + panicOn(err) + + rbm.UnionInPlace(others...) + // iterate over the containers that changed within rbm, and write them back to disk. + + it, found := rbm.Containers.Iterator(0) + _ = found // don't care about the value of found, because first containerKey might be > 0 + + for it.Next() { + containerKey, rc := it.Value() + + // TODO: only write the changed ones back, as optimization? + // Compare to ImportRoaringBits. + err := tx.PutContainer(index, field, view, shard, containerKey, rc) + panicOn(err) + } + return nil +} + +// CountRange returns the count of hot bits in the start, end range on the fragment. +// roaring.countRange counts the number of bits set between [start, end). +func (tx *BoltTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { + + if start >= end { + return 0, nil + } + + skey := highbits(start) + ekey := highbits(end) + + citer, found, err := tx.ContainerIterator(index, field, view, shard, skey) + _ = found + panicOn(err) + + defer citer.Close() + + // If range is entirely in one container then just count that range. + if skey == ekey { + citer.Next() + _, c := citer.Value() + return uint64(c.CountRange(int32(lowbits(start)), int32(lowbits(end)))), nil + } + + for citer.Next() { + k, c := citer.Value() + if k < skey { + citer.Close() + panic(fmt.Sprintf("should be impossible for k(%v) to be less than skey(%v). tx p=%p", k, skey, tx)) + } + + // k > ekey handles the case when start > end and where start and end + // are in different containers. Same container case is already handled above. + if k > ekey { + break + } + if k == skey { + n += uint64(c.CountRange(int32(lowbits(start)), roaring.MaxContainerVal+1)) + continue + } + if k < ekey { + n += uint64(c.N()) + continue + } + if k == ekey { + n += uint64(c.CountRange(0, int32(lowbits(end)))) + break + } + } + + return n, nil +} + +// OffsetRange creates a new roaring.Bitmap to return in other. For all the +// hot bits in [start, endx) of the chosen fragment, it stores +// them into other but with offset added to their bit position. +// The primary client is doing this, using ShardWidth, already; see +// fragment.rowFromStorage() in fragment.go. For example: +// +// data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard, +// +// f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) +// ^ offset ^ start ^ endx +// +// The start and endx arguments are container keys that have been shifted left by 16 bits; +// their highbits() will be taken to determine the actual container keys. This +// is done to conform to the roaring.OffsetRange() argument convention. +// +func (tx *BoltTx) OffsetRange(index, field, view string, shard, offset, start, endx uint64) (other *roaring.Bitmap, err error) { + ////vv("top of BoltTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v)", index, field, view, int(shard), int(offset), int(start), int(endx)) + //defer func() { + ////vv("returning from BoltTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) other returning is: '%#v' stack=\n%v", index, field, view, int(shard), int(offset), int(start), int(endx), asInts(other.Slice()), stack()) + //}() + + // roaring does these three checks in its OffsetRange + if lowbits(offset) != 0 { + panic("offset must not contain low bits") + } + if lowbits(start) != 0 { + panic("range start must not contain low bits") + } + if lowbits(endx) != 0 { + panic("range end must not contain low bits") + } + + other = roaring.NewSliceBitmap() + off := highbits(offset) + hi0, hi1 := highbits(start), highbits(endx) + + needle := txkey.Key(index, field, view, shard, hi0) + prefix := txkey.Prefix(index, field, view, shard) + + it := NewBoltIterator(tx, prefix) + defer it.Close() + it.Seek(needle) + for ; it.ValidForPrefix(prefix); it.Next() { + bkey := it.lastKey + k := txkey.KeyExtractContainerKey(bkey) + + // >= hi1 is correct b/c endx cannot have any lowbits set. + if uint64(k) >= hi1 { + break + } + destCkey := off + (k - hi0) + + v := it.lastVal + n := len(v) + if n == 0 { + continue + } + c := tx.toContainer(v[n-1], v[0:(n-1)]) + other.Containers.Put(destCkey, c.Freeze()) + } + return other, nil +} + +// IncrementOpN increments the tx opcount by changedN +func (tx *BoltTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { + tx.opcount += changedN +} + +// ImportRoaringBits handles deletes by setting clear=true. +// rowSet[rowID] returns the number of bit changed on that rowID. +func (tx *BoltTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { + + n := itr.Len() + if n == 0 { + return + } + rowSet = make(map[uint64]int) + + var currRow uint64 + + var oldC *roaring.Container + for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { + if rowSize != 0 { + currRow = itrKey / rowSize + } + nsynth := int(synthC.N()) + if nsynth == 0 { + continue + } + // INVAR: nsynth > 0 + + oldC, err = tx.Container(index, field, view, shard, itrKey) + panicOn(err) + if err != nil { + return + } + + if oldC == nil || oldC.N() == 0 { + // no container at the itrKey in bolt (or all zero container). + if clear { + // changed of 0 and empty rowSet is perfect, no need to change the defaults. + continue + } else { + + changed += nsynth + rowSet[currRow] += nsynth + + err = tx.PutContainer(index, field, view, shard, itrKey, synthC) + if err != nil { + return + } + continue + } + } + + if clear { + existN := oldC.N() // number of bits set in the old container + newC := oldC.Difference(synthC) + + // update rowSet and changes + if newC.N() == existN { + // INVAR: do changed need adjusting? nope. same bit count, + // so no change could have happened. + continue + } else { + changes := int(existN - newC.N()) + changed += changes + rowSet[currRow] -= changes + + if tx.DeleteEmptyContainer && newC.N() == 0 { + err = tx.RemoveContainer(index, field, view, shard, itrKey) + if err != nil { + return + } + continue + } + err = tx.PutContainer(index, field, view, shard, itrKey, newC) + if err != nil { + return + } + continue + } + } else { + // setting bits + + existN := oldC.N() + if existN == roaring.MaxContainerVal+1 { + // completely full container already, set will do nothing. so changed of 0 default is perfect. + continue + } + if existN == 0 { + // can nsynth be zero? No, because of the continue/invariant above where nsynth > 0 + changed += nsynth + rowSet[currRow] += nsynth + err = tx.PutContainer(index, field, view, shard, itrKey, synthC) + if err != nil { + return + } + continue + } + + newC := roaring.Union(oldC, synthC) // UnionInPlace was giving us crashes on overly large containers. + + if roaring.ContainerType(newC) == roaring.ContainerBitmap { + newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it. + } + if newC.N() != existN { + changes := int(newC.N() - existN) + changed += changes + rowSet[currRow] += changes + + err = tx.PutContainer(index, field, view, shard, itrKey, newC) + if err != nil { + panicOn(err) + return + } + continue + } + } + } + return +} + +func (tx *BoltTx) toContainer(typ byte, v []byte) (r *roaring.Container) { + + //tx.debugOnlyGidcheck() + + if len(v) == 0 { + return nil + } + + var w []byte + useRowCache := tx.UseRowCache() + if tx.doAllocZero || useRowCache { + // Do electric fence-inspired bad-memory read detection. + // + // The v []byte lives in BoltDB's memory-mapped vlog-file, + // and Bolt will recycle it after tx ends with rollback or commit. + // + // Problem is, at least some operations were not respecting transaction boundaries. + // This technique helped us find them. The rowCache was an example. + // + // See the global const DetectMemAccessPastTx + // at the top of txfactory.go to activate/deactivate this. + // + // Seebs suggested this nice variation: we could use individual mmaps for these + // copies, which would be unusable in production, but workable for testing, and then unmap them, + // which would get us probable segfaults on future accesses to them. + // + // The go runtime also has an -efence flag which may be similarly useful if really pressed. + // + w = make([]byte, len(v)) + copy(w, v) + } else { + w = v + } + return ToContainer(typ, w) +} + +// StringifiedBoltKeys returns a string with all the container +// keys available in bolt. +func (w *BoltWrapper) StringifiedBoltKeys(optionalUseThisTx Tx, short bool) (r string) { + if optionalUseThisTx == nil { + tx, _ := w.NewTx(!writable, "", Txo{}) + defer tx.Rollback() + r = stringifiedBoltKeysTx(tx.(*BoltTx), short) + return + } + + btx, ok := optionalUseThisTx.(*BoltTx) + if !ok { + return fmt.Sprintf("", optionalUseThisTx) + } + r = stringifiedBoltKeysTx(btx, short) + return +} + +// countBitsSet returns the number of bits set (or "hot") in +// the roaring container value found by the txkey.Key() +// formatted bkey. +func (tx *BoltTx) countBitsSet(bkey []byte) (n int) { + + //tx.debugOnlyGidcheck() + + bkt := tx.tx.Bucket(bucketCT) + v := bkt.Get(bkey) + + if v == nil { + // some queries bkey may not be present! don't panic. + return 0 + } + + n = len(v) + if n > 0 { + rc := tx.toContainer(v[n-1], v[0:(n-1)]) + n = int(rc.N()) + } + return +} + +func (tx *BoltTx) Dump(short bool, shard uint64) { + fmt.Printf("%v\n", stringifiedBoltKeysTx(tx, short)) +} + +// stringifiedBoltKeysTx reports all the bolt keys and a +// corresponding blake3 hash viewable by txn within the entire +// bolt database. +// It also reports how many bits are hot in the roaring container +// (how many bits are set, or 1 rather than 0). +// +// By convention, we must return the empty string if there +// are no keys present. The tests use this to confirm +// an empty database. +func stringifiedBoltKeysTx(tx *BoltTx, short bool) (r string) { + + r = "allkeys:[\n" + it := NewBoltIterator(tx, nil) + defer it.Close() + any := false + for it.Next() { + any = true + + bkey := it.lastKey + key := txkey.ToString(bkey) + ckey := txkey.KeyExtractContainerKey(bkey) + h := "" + srbm := "" + v := it.lastVal + n := len(v) + if n == 0 { + panic("should not have empty v here") + } + h = hash.Blake3sum16(v[0:(n - 1)]) + ct := tx.toContainer(v[n-1], v[0:(n-1)]) + cts := roaring.NewSliceContainers() + cts.Put(ckey, ct) + rbm := &roaring.Bitmap{Containers: cts} + srbm = BitmapAsString(rbm) + + r += fmt.Sprintf("%v -> %v (%v hot)\n", key, h, tx.countBitsSet(bkey)) + if !short { + r += " ......." + srbm + "\n" + } + } + r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + + if !any { + return "" + } + return "bolt-" + r +} + +func (w *BoltWrapper) DeleteDBPath(dbs *DBShard) (err error) { + path := dbs.pathForType(boltTxn) + err = os.RemoveAll(path) + if err != nil { + return errors.Wrap(err, "DeleteDBPath") + } + return +} + +func (w *BoltWrapper) DeleteField(index, field, fieldPath string) (err error) { + + // TODO(jea) cleanup: I think this fieldPath delete just goes away now. + // remove this commented stuff once we are sure. + // + // under blue-green roaring_bolt, the directory will not be found, b/c roaring will have + // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: + // "If the path does not exist, RemoveAll returns nil (no error)" + + err = os.RemoveAll(fieldPath) + if err != nil { + return errors.Wrap(err, "removing directory") + } + prefix := txkey.FieldPrefix(index, field) + return w.DeletePrefix(prefix) +} + +func (w *BoltWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { + prefix := txkey.Prefix(index, field, view, shard) + return w.DeletePrefix(prefix) +} + +func (w *BoltWrapper) DeletePrefix(prefix []byte) error { + + tx, _ := w.NewTx(writable, w.name, Txo{}) + + // NewTx will grab these, so don't lock until after it. + w.muDb.Lock() + + bi := NewBoltIterator(tx.(*BoltTx), prefix) + + for bi.Next() { + //vv("deleting next in cur") + err := bi.cur.Delete() + if err != nil { + w.muDb.Unlock() + panic(err) + } + } + bi.Close() + + // Commit will grab the w.muDb lock, so we must release it first. + w.muDb.Unlock() + + err := tx.Commit() + panicOn(err) + + return nil +} + +func (tx *BoltTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { + + rbm, err := tx.RoaringBitmap(index, field, view, shard) + if err != nil { + return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") + } + var buf bytes.Buffer + sz, err = rbm.WriteTo(&buf) + if err != nil { + return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") + } + return ioutil.NopCloser(&buf), sz, err +} + +func (tx *BoltTx) Options() Txo { + return tx.o +} + +// Sn retreives the serial number of the Tx. +func (tx *BoltTx) Sn() int64 { + return tx.sn +} diff --git a/badger_test.go b/bolt_test.go similarity index 57% rename from badger_test.go rename to bolt_test.go index a4ad6ab71..1030a5fe2 100644 --- a/badger_test.go +++ b/bolt_test.go @@ -12,15 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// explanation of build tags: -// -// badgerdb builds but won't run in 32-bit 386 world, as of 2020 July 20. -// See https://github.com/dgraph-io/badger/issues/1384 for any progress. -// What we see is that the value-log allocations immediately run out of -// memory. So we turn off 386 with a build tag to keep the .circleci happy. - -// +build !386 - package pilosa import ( @@ -29,22 +20,13 @@ import ( "os" "testing" - "github.com/dgraph-io/badger/v2" "github.com/pilosa/pilosa/v2/roaring" - "github.com/pilosa/pilosa/v2/testhook" - "github.com/pkg/errors" ) -var _ = &roaring.Bitmap{} - -func init() { - testhook.RegisterPostTestHook(reportTestBadgersNeedingClose) -} - // helpers, each runs their own new txn, and commits if a change/delete // was made. The txn is rolled back if it is just viewing the data. -func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) { +func BoltMustHaveBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, bitvalue uint64) { tx, _ := dbwrap.NewTx(!writable, index, Txo{}) defer tx.Rollback() @@ -57,7 +39,7 @@ func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string tx.Rollback() } -func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) { +func BoltMustNotHaveBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, bitvalue uint64) { tx, _ := dbwrap.NewTx(!writable, index, Txo{}) defer tx.Rollback() @@ -69,7 +51,7 @@ func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view str tx.Rollback() } -func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) { +func BoltMustSetBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { tx, _ := dbwrap.NewTx(writable, index, Txo{}) // add a bit @@ -87,37 +69,37 @@ func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, panicOn(tx.Commit()) } -func badgerDBMustDeleteBitvalueContainer(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) { +func BoltMustDeleteBitvalueContainer(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { tx, _ := dbwrap.NewTx(writable, index, Txo{}) hi := highbits(putme) panicOn(tx.RemoveContainer(index, field, view, shard, hi)) panicOn(tx.Commit()) } -func badgerDBMustDeleteBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) { +func BoltMustDeleteBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { tx, _ := dbwrap.NewTx(writable, index, Txo{}) _, err := tx.Remove(index, field, view, shard, putme) panicOn(err) panicOn(tx.Commit()) } -func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func()) { +func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) { var err error - fn := badgerPath(path) + fn := boltPath(path) panicOn(os.RemoveAll(fn)) - ww, err := globalBadgerReg.OpenDBWrapper(path, DetectMemAccessPastTx) + ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx) panicOn(err) - w = ww.(*BadgerDBWrapper) + w = ww.(*BoltWrapper) // verify it is empty - allkeys := w.StringifiedBadgerKeys(nil, true) - if allkeys != "" { + allkeys := w.StringifiedBoltKeys(nil, false) + if allkeys != "" { panic(fmt.Sprintf("freshly created database was not empty! had keys:'%v'", allkeys)) } return w, func() { - w.Close() // stop any started background GC goroutine. - os.RemoveAll(fn) + w.Close() + panicOn(os.RemoveAll(fn)) } } @@ -127,10 +109,10 @@ func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func() ////////////////////////// // begin Tx method tests -func TestBadger_DeleteFragment(t *testing.T) { +func TestBolt_DeleteFragment(t *testing.T) { // setup - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteFragment") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_DeleteFragment") defer clean() defer dbwrap.Close() index, field, view, shard0 := "i", "f", "v", uint64(0) @@ -189,35 +171,70 @@ func TestBadger_DeleteFragment(t *testing.T) { } } -func TestBadger_Max_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Max_on_many_containers") +func TestBolt_Max_on_many_containers(t *testing.T) { + path := "TestBolt_Max_on_many_containers" + dbwrap, clean := mustOpenEmptyBoltWrapper(path) + defer clean() defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) + index, field, view := "i", "f", "v" - putmeValues := []uint64{0, 2 << 16, 4 << 16} + // 099 + // 101 + // 199 + // 300 + // 399 + // + // find max in [300,400) and get 399 + // find max in [000,100) and get 099 + // find max in [100,200) and get 199 + // find max in [400,500) and get nothing back + // find max in [200,300) and get nothing back - for _, putme := range putmeValues { - badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + shards := []int{99, 101, 199, 300, 399} + + for _, sh := range shards { + shard := uint64(sh) + for _, pm := range shards { + putme := uint64(pm) + if putme > shard { + continue + } + BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + } } tx, _ := dbwrap.NewTx(!writable, index, Txo{}) defer tx.Rollback() - max, err := tx.Max(index, field, view, shard) - panicOn(err) - expected := putmeValues[len(putmeValues)-1] - if max != expected { - panic(fmt.Sprintf("expected Max() of %v but got max=%v", expected, max)) + for _, shard := range shards { + max, err := tx.Max(index, field, view, uint64(shard)) + panicOn(err) + if max != uint64(shard) { + panic(fmt.Sprintf("expected max (%v) to be == shard = %v", max, shard)) + } } + + // check for not found + max, err := tx.Max(index, field, view, uint64(200)) + panicOn(err) + if max != 0 { + panic("expected not found to give 0 max back with nil err") + } + max, err = tx.Max(index, field, view, uint64(400)) + panicOn(err) + if max != 0 { + panic("expected not found to give 0 max back with nil err") + } + } // and the rest -func TestBadger_SetBitmap(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SetBitmap") +func TestBolt_SetBitmap(t *testing.T) { + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_SetBitmap") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -257,8 +274,8 @@ func TestBadger_SetBitmap(t *testing.T) { tx2.Rollback() } -func TestBadger_OffsetRange(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_OffsetRange") +func TestBolt_OffsetRange(t *testing.T) { + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_OffsetRange") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -324,8 +341,8 @@ func TestBadger_OffsetRange(t *testing.T) { } } -func TestBadger_Count_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Count_on_many_containers") +func TestBolt_Count_on_many_containers(t *testing.T) { + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Count_on_many_containers") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -333,9 +350,9 @@ func TestBadger_Count_on_many_containers(t *testing.T) { putmeValues := []uint64{0, 2 << 16, 4 << 16} for _, putme := range putmeValues { - badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } tx, _ := dbwrap.NewTx(writable, index, Txo{}) @@ -348,8 +365,8 @@ func TestBadger_Count_on_many_containers(t *testing.T) { } } -func TestBadger_Count_dense_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Count_dense_containers") +func TestBolt_Count_dense_containers(t *testing.T) { + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Count_dense_containers") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -357,9 +374,6 @@ func TestBadger_Count_dense_containers(t *testing.T) { tx, _ := dbwrap.NewTx(writable, index, Txo{}) expected := 0 - // can't do more than about 100k writes per badger txn by default, so - // have to keep this kind of small. - // (See maxBatchCount:104857, maxBatchSize:10066329). for i := uint64(0); i < (1<<16)+2; i += 2 { changed, err := tx.Add(index, field, view, shard, doBatched, i) panicOn(err) @@ -377,9 +391,9 @@ func TestBadger_Count_dense_containers(t *testing.T) { } } -func TestBadger_ContainerIterator_on_empty(t *testing.T) { +func TestBolt_ContainerIterator_on_empty(t *testing.T) { // iterate on empty container, should not find anything. - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -395,9 +409,9 @@ func TestBadger_ContainerIterator_on_empty(t *testing.T) { panicOn(err) } -func TestBadger_ContainerIterator_on_one_bit(t *testing.T) { +func TestBolt_ContainerIterator_on_one_bit(t *testing.T) { // set one bit, iterate. - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_on_one_bit") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -453,8 +467,8 @@ func TestBadger_ContainerIterator_on_one_bit(t *testing.T) { } } -func TestBadger_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit") +func TestBolt_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) { + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_on_one_bit") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -508,8 +522,8 @@ func TestBadger_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) { panicOn(err) } -func TestBadger_ContainerIterator_empty_iteration_loop(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_empty_iteration_loop") +func TestBolt_ContainerIterator_empty_iteration_loop(t *testing.T) { + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_empty_iteration_loop") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -557,9 +571,9 @@ func TestBadger_ContainerIterator_empty_iteration_loop(t *testing.T) { } -func TestBadger_ForEach_on_one_bit(t *testing.T) { +func TestBolt_ForEach_on_one_bit(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_on_one_bit") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -596,9 +610,9 @@ func TestBadger_ForEach_on_one_bit(t *testing.T) { } } -func TestBadger_RemoveContainer_one_bit_test(t *testing.T) { +func TestBolt_RemoveContainer_one_bit_test(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_RemoveContainer_one_bit_test") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_RemoveContainer_one_bit_test") defer clean() defer dbwrap.Close() @@ -610,16 +624,17 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) { // a) delete of whole container in a seperate txn. Commit should establish the deletion. - badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) + + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) + BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) // b) deletion + rollback on the txn should restore the deleted bit - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) // delete, but rollback instead of commit tx, _ := dbwrap.NewTx(writable, index, Txo{}) @@ -628,7 +643,7 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) { tx.Rollback() // verify that the rollback undid the deletion. - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) // c) within one Tx, after delete it should be gone as viewed within the txn. tx, _ = dbwrap.NewTx(writable, index, Txo{}) @@ -651,15 +666,15 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) { tx.Rollback() // verify that the rollback undid the deletion. - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) // leave with clean slate - badgerDBMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) + BoltMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) } } -func TestBadger_Remove_one_bit_test(t *testing.T) { +func TestBolt_Remove_one_bit_test(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Remove_one_bit_test") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Remove_one_bit_test") defer clean() defer dbwrap.Close() @@ -671,16 +686,16 @@ func TestBadger_Remove_one_bit_test(t *testing.T) { // a) delete of whole container in a seperate txn. Commit should establish the deletion. - badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustDeleteBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustDeleteBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) // b) deletion + rollback on the txn should restore the deleted bit - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) // delete, but rollback instead of commit tx, _ := dbwrap.NewTx(writable, index, Txo{}) @@ -691,7 +706,7 @@ func TestBadger_Remove_one_bit_test(t *testing.T) { tx.Rollback() // verify that the rollback undid the deletion. - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) // c) within one Tx, after delete it should be gone as viewed within the txn. tx, _ = dbwrap.NewTx(writable, index, Txo{}) @@ -713,64 +728,14 @@ func TestBadger_Remove_one_bit_test(t *testing.T) { tx.Rollback() // verify that the rollback undid the deletion. - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) // leave with clean slate - badgerDBMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) + BoltMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) } } -func TestBadger_reverse_badger_iterator(t *testing.T) { - - // sanity check our understanding of Seek()-ing on reverse iterators. - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_reverse_badger_iterator") - defer clean() - defer dbwrap.Close() - - // add 0, 1, 2 to badgerdb as keys (and same value). - err := dbwrap.Update(func(txn *badger.Txn) error { - for i := 0; i < 3; i++ { - kv := []byte(fmt.Sprintf("a:%v", i)) - err := txn.Set(kv, kv) - panicOn(err) - } - return nil - }) - panicOn(err) - - tx := dbwrap.db.NewTransactionAt(1, !writable) - - opts := badger.DefaultIteratorOptions - opts.PrefetchValues = false // else by default, pre-fetches the 1st 100 values, which would be slow. - opts.Reverse = true - it := tx.NewIterator(opts) - it.Rewind() - if !it.Valid() { - panic("invalid reversed iterator?") - } - a := []byte("a:3") - it.Seek(a) - if !it.Valid() { - panic("invalid reversed iterator after seek") - } - - it.Next() - if !it.Valid() { - panic("invalid reversed iterator after seek and next") - } - item := it.Item() - - err = item.Value(func(val []byte) error { - // This func with val would only be called if item.Value encounters no error. - if string(val) != "a:1" { - panic(fmt.Sprintf("we are in trouble, should have gotten 'a:1' but instead got '%v'", string(val))) - } - return nil - }) - panicOn(err) -} - -func TestBadger_Min_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Min_on_many_containers") +func TestBolt_Min_on_many_containers(t *testing.T) { + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Min_on_many_containers") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -788,9 +753,9 @@ func TestBadger_Min_on_many_containers(t *testing.T) { putmeValues := []uint64{3, 2 << 16, 4 << 16} for _, putme := range putmeValues { - badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } tx, _ = dbwrap.NewTx(!writable, index, Txo{}) @@ -807,8 +772,8 @@ func TestBadger_Min_on_many_containers(t *testing.T) { } } -func TestBadger_CountRange_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_CountRange_on_many_containers") +func TestBolt_CountRange_on_many_containers(t *testing.T) { + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_CountRange_on_many_containers") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -825,9 +790,9 @@ func TestBadger_CountRange_on_many_containers(t *testing.T) { putmeValues := []uint64{3, 2 << 16, 4 << 16} for _, putme := range putmeValues { - badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } tx, _ = dbwrap.NewTx(!writable, index, Txo{}) @@ -844,8 +809,8 @@ func TestBadger_CountRange_on_many_containers(t *testing.T) { } } -func TestBadger_CountRange_middle_container(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_CountRange_middle_container") +func TestBolt_CountRange_middle_container(t *testing.T) { + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_CountRange_middle_container") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -853,9 +818,9 @@ func TestBadger_CountRange_middle_container(t *testing.T) { putmeValues := []uint64{3, 2 << 16, 4 << 16} for _, putme := range putmeValues { - badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } tx, _ := dbwrap.NewTx(!writable, index, Txo{}) @@ -869,8 +834,8 @@ func TestBadger_CountRange_middle_container(t *testing.T) { } } -func TestBadger_CountRange_many_middle_container(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_CountRange_many_middle_container") +func TestBolt_CountRange_many_middle_container(t *testing.T) { + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_CountRange_many_middle_container") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -878,9 +843,9 @@ func TestBadger_CountRange_many_middle_container(t *testing.T) { putmeValues := []uint64{3, 2 << 16, 4 << 16} for _, putme := range putmeValues { - badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } tx, _ := dbwrap.NewTx(!writable, index, Txo{}) @@ -894,9 +859,9 @@ func TestBadger_CountRange_many_middle_container(t *testing.T) { } } -func TestBadger_UnionInPlace(t *testing.T) { +func TestBolt_UnionInPlace(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_UnionInPlace") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_UnionInPlace") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -909,9 +874,9 @@ func TestBadger_UnionInPlace(t *testing.T) { // populate others with putmeValues +1 into others for _, putme := range putmeValues { - badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) @@ -943,16 +908,16 @@ func TestBadger_UnionInPlace(t *testing.T) { } } -func TestBadger_RoaringBitmap(t *testing.T) { +func TestBolt_RoaringBitmap(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_RoaringBitmap") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_RoaringBitmap") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) expected := uint64(3) putme := expected - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) tx, _ := dbwrap.NewTx(!writable, index, Txo{}) defer tx.Rollback() @@ -966,123 +931,15 @@ func TestBadger_RoaringBitmap(t *testing.T) { } } -func TestBadger_reverse_badger_iterator_and_prefix_valid(t *testing.T) { +func TestBolt_ImportRoaringBits(t *testing.T) { - // does a reverse iterator and ValidForPrefix behave like we expect it too? - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_reverse_badger_iterator_and_prefix_valid") - defer clean() - defer dbwrap.Close() - - // add 0, 1, 2 to badgerdb as keys (and same value). - err := dbwrap.Update(func(txn *badger.Txn) error { - for _, prefix := range []string{"a", "b", "c"} { - for i := 0; i < 3; i++ { - kv := []byte(fmt.Sprintf("%v:%v", prefix, i)) - err := txn.Set(kv, kv) - panicOn(err) - } - } - return nil - }) - panicOn(err) - - tx0, _ := dbwrap.NewTx(!writable, "no-index-avail", Txo{}) - tx := tx0.(*BadgerTx) - - prefix := []byte("b:") - it := NewBadgerIterator(tx, prefix) - - if !it.it.Valid() { - panic("why is underlying badger it not valid here?") - } - results := "" - for it.Next() { - item := it.it.Item() - sk := string(item.Key()) - results += sk + ", " - } - expected := `b:0, b:1, b:2, ` - if results != expected { - panic(fmt.Sprintf("observed: '%v' but expected: '%v'", results, expected)) - } - it.Close() - - // now reversed - seekto := []byte("c:") - rit := NewBadgerReverseIterator(tx, prefix, seekto) // Seeks("b:") goes to b:0 - defer rit.Close() - - if !rit.it.Valid() { - panic("why is underlying badger it not valid here?") - } - results = "" - for rit.Next() { - item := rit.it.Item() - sk := string(item.Key()) - results += sk + ", " - } - expected = `b:2, b:1, b:0, ` - if results != expected { - panic(fmt.Sprintf("observed: '%v' but expected: '%v'", results, expected)) - } - rit.Close() -} - -func TestBadger_just_reverse_badger_iterator_and_prefix_valid(t *testing.T) { - - // does a reverse iterator and ValidForPrefix behave like we expect it too? - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_reverse_badger_iterator_and_prefix_valid") - defer clean() - defer dbwrap.Close() - - // add 0, 1, 2 to badgerdb as keys (and same value). - err := dbwrap.Update(func(txn *badger.Txn) error { - for _, prefix := range []string{"a", "b", "c"} { - for i := 0; i < 3; i++ { - kv := []byte(fmt.Sprintf("%v:%v", prefix, i)) - err := txn.Set(kv, kv) - panicOn(err) - } - } - return nil - }) - panicOn(err) - - tx0, _ := dbwrap.NewTx(!writable, "no-index-avail", Txo{}) - tx := tx0.(*BadgerTx) - - seekto := []byte("c:") - prefix := []byte("b:") - // now reversed - rit := NewBadgerReverseIterator(tx, prefix, seekto) // Seeks("b:") goes to b:0 - defer rit.Close() - - if !rit.it.Valid() { - panic("why is underlying badger rit not valid here?") - } - results := "" - for rit.Next() { - item := rit.it.Item() - sk := string(item.Key()) - results += sk + ", " - } - expected := `b:2, b:1, b:0, ` - if results != expected { - panic(fmt.Sprintf("observed: '%v' but expected: '%v'", results, expected)) - } - rit.Close() -} - -func TestBadger_ImportRoaringBits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ImportRoaringBits") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) - tx0, _ := dbwrap.NewTx(writable, index, Txo{}) - tx := tx0.(*BadgerTx) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) defer tx.Rollback() - tx.DeleteEmptyContainer = true // traditional badger Tx behavior, but not Roaring. + tx.(*BoltTx).DeleteEmptyContainer = true // traditional lmdb Tx behavior, but not Roaring. //bitvalue := uint64(42) @@ -1149,17 +1006,17 @@ func TestBadger_ImportRoaringBits(t *testing.T) { if n != 0 { panic(fmt.Sprintf("n = %v not zero so the clearbits didn't happen!", n)) } - allkeys := stringifiedBadgerKeysTx(tx, false) + allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) // should have no keys - if allkeys != "" { - panic("badger should have no keys now") + if allkeys != "" { + panic("bolt should have no keys now") } } -func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { +func TestBolt_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits_set_nonoverlapping_bits") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ImportRoaringBits_set_nonoverlapping_bits") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -1207,9 +1064,9 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { panicOn(err) } -func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { +func TestBolt_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits_clear_nonoverlapping_bits") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ImportRoaringBits_clear_nonoverlapping_bits") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) @@ -1265,74 +1122,14 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { } -/* -func TestBadger_AutoCommit(t *testing.T) { +func TestBolt_DeleteIndex(t *testing.T) { // setup - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_AutoCommit") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_DeleteIndex") defer clean() defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - // if we go over 100K writes, we should autocommit - // rather than panic. - for v := 0; v < 133444; v++ { - changed, err := tx.Add(index, field, view, shard, doBatched, uint64(v)) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - } - - err := tx.Commit() - panicOn(err) -} - -func TestBadger_BigWritesAvoidTxnTooLargeWithAutoCommit(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_BigWritesAvoidTxnTooLargeWithAutoCommit") - defer clean() - defer dbwrap.Close() - - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - containerKey := uint64(0) - // setup - bits := make([]uint64, 1024) - n := 0 - for i := range bits { - bits[i] = ^uint64(0) - n += 64 - } - rc := roaring.NewContainerBitmap(n, bits) - - // if we go over 100K big writes, we should autocommit - // rather than panic. - for v := 0; v < 133444; v++ { - containerKey++ - err := tx.PutContainer(index, field, view, shard, containerKey, rc) - panicOn(err) - } - - err := tx.Commit() - panicOn(err) -} -*/ - -func TestBadger_DeleteIndex(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx0, _ := dbwrap.NewTx(writable, index, Txo{}) - tx := tx0.(*BadgerTx) - bitvalue := uint64(777) bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} for _, v := range bits { @@ -1369,9 +1166,7 @@ func TestBadger_DeleteIndex(t *testing.T) { err = dbwrap.DeleteIndex(index) panicOn(err) - tx0, _ = dbwrap.NewTx(!writable, index2, Txo{}) - tx = tx0.(*BadgerTx) - + tx, _ = dbwrap.NewTx(!writable, index2, Txo{}) defer tx.Rollback() exists, err = tx.Contains(index2, field, view, shard, bitvalue) panicOn(err) @@ -1383,22 +1178,20 @@ func TestBadger_DeleteIndex(t *testing.T) { exists, err = tx.Contains(index, field, view, shard, v) panicOn(err) if exists { - allkeys := stringifiedBadgerKeysTx(tx, false) + allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) } } } -func TestBadger_DeleteIndex_over100k(t *testing.T) { - t.Skip("test big and long running, skip") +func TestBolt_DeleteIndex_over100k(t *testing.T) { + // setup - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex_over100k") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_DeleteIndex_over100k") defer clean() defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) - tx0, _ := dbwrap.NewTx(writable, index, Txo{}) - tx := tx0.(*BadgerTx) - + tx, _ := dbwrap.NewTx(writable, index, Txo{}) bitvalue := uint64(777) limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction. //limit := uint64(101) @@ -1411,8 +1204,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) { panicOn(err) if v%100000 == 0 { panicOn(tx.Commit()) - tx0, _ := dbwrap.NewTx(writable, index, Txo{}) - tx = tx0.(*BadgerTx) + tx, _ = dbwrap.NewTx(writable, index, Txo{}) } } @@ -1429,9 +1221,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) { err = dbwrap.DeleteIndex(index) panicOn(err) - tx0, _ = dbwrap.NewTx(!writable, index2, Txo{}) - tx = tx0.(*BadgerTx) - + tx, _ = dbwrap.NewTx(!writable, index2, Txo{}) defer tx.Rollback() exists, err := tx.Contains(index2, field, view, shard, bitvalue) panicOn(err) @@ -1443,93 +1233,22 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) { exists, err = tx.Contains(index, field, view, shard, v<<16) panicOn(err) if exists { - allkeys := stringifiedBadgerKeysTx(tx, false) + allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) } } } -func TestBitmapDiff(t *testing.T) { - a := roaring.NewBitmap() - b := roaring.NewBitmap() - err := roaringBitmapDiff(a, b) - panicOn(err) - err = roaringBitmapDiff(b, a) - panicOn(err) +func TestBolt_SliceOfShards(t *testing.T) { - a = roaring.NewBitmap(0) - err = roaringBitmapDiff(a, b) - if err == nil { - panic("diff should have been noticed") - } - err = roaringBitmapDiff(b, a) - if err == nil { - panic("diff should have been noticed") - } - - b = roaring.NewBitmap(0) - err = roaringBitmapDiff(a, b) - panicOn(err) - err = roaringBitmapDiff(b, a) - panicOn(err) - - a = roaring.NewBitmap() - - err = roaringBitmapDiff(a, b) - if err == nil { - panic("diff should have been noticed") - } - err = roaringBitmapDiff(b, a) - if err == nil { - panic("diff should have been noticed") - } - - a = roaring.NewBitmap(1) - - err = roaringBitmapDiff(a, b) - if err == nil { - panic("diff should have been noticed") - } - err = roaringBitmapDiff(b, a) - if err == nil { - panic("diff should have been noticed") - } - - b = roaring.NewBitmap(1, 2) - a = roaring.NewBitmap(0, 1) - - err = roaringBitmapDiff(a, b) - if err == nil { - panic("diff should have been noticed") - } - err = roaringBitmapDiff(b, a) - if err == nil { - panic("diff should have been noticed") - } - - b = roaring.NewBitmap(1, 2, 3) - a = roaring.NewBitmap(1, 2) - - err = roaringBitmapDiff(a, b) - if err == nil { - panic("diff should have been noticed") - } - err = roaringBitmapDiff(b, a) - if err == nil { - panic("diff should have been noticed") - } -} - -func TestBadger_SliceOfShards(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SliceOfShards") + dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_SliceOfShards") defer clean() defer dbwrap.Close() index, field, view := "i", "f", "v" shards := []uint64{0, 1, 2, 3, 1000001, 2000001} putme := uint64(179) for _, shard := range shards { - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) } tx, _ := dbwrap.NewTx(!writable, index, Txo{}) defer tx.Rollback() @@ -1543,9 +1262,9 @@ func TestBadger_SliceOfShards(t *testing.T) { } } -func TestBadger_HasData(t *testing.T) { +func TestBolt_HasData(t *testing.T) { - db, clean := mustOpenEmptyBadgerWrapper("TestBadger_HasData") + db, clean := mustOpenEmptyBoltWrapper("TestBolt_SliceOfShards") defer clean() defer db.Close() @@ -1561,7 +1280,7 @@ func TestBadger_HasData(t *testing.T) { // check that HasData sees a committed record. index, field, view, shard, putme := "i", "f", "v", uint64(123), uint64(42) - badgerDBMustSetBitvalue(db, index, field, view, shard, putme) + BoltMustSetBitvalue(db, index, field, view, shard, putme) // HasData(false) should now report data hasAnything, err = db.HasData() @@ -1572,185 +1291,3 @@ func TestBadger_HasData(t *testing.T) { t.Fatalf("HasData() reported no data on a database that has bits written to it") } } - -// Benchmark performance of setValue for BSI ranges. -func BenchmarkBadger_Write(b *testing.B) { - - dbwrap, clean := mustOpenEmptyBadgerWrapper("BenchmarkBadger_Write") - //defer clean() - _ = clean - defer dbwrap.Close() - - putmeValues := []uint64{3, 2 << 16} - index, field, view, shard := "i", "f", "v", uint64(0) - - for _, putme := range putmeValues { - badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - } - /* - - dbwrap, clean := mustOpenEmptyBadgerWrapper("BenchmarkBadger_Write") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - bitvalue := uint64(1 << 20) - changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - - bitvalue2 := uint64(1<<20 + 1) - changed, err = tx.Add(index, field, view, shard, doBatched, bitvalue2) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - exists, err = tx.Contains(index, field, view, shard, bitvalue2) - panicOn(err) - if !exists { - panic("ARG bitvalue2 was NOT SET!!!") - } - - err = tx.Commit() - panicOn(err) - - offset := uint64(0 << 20) - start := uint64(0 << 16) - endx := bitvalue + 1<<16 - - tx2 := dbwrap.NewTx(!writable, index, Txo{}) - rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx) - panicOn(err) - tx2.Rollback() - - // should see our 1M value - s2 := bitmapAsString(rbm2) - expect2 := "c(1048576, 1048577)" - if s2 != expect2 { - panic(fmt.Sprintf("s2='%v', but expected '%v'", s2, expect2)) - } - - // now offset by 2M - offset = uint64(2 << 20) - tx3 := dbwrap.NewTx(!writable, index, Txo{}) - rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx) - panicOn(err) - tx3.Rollback() - - //expect to see 3M == 3145728 - s3 := bitmapAsString(rbm3) - expect3 := "c(3145728, 3145729)" - - if s3 != expect3 { - panic(fmt.Sprintf("s3='%v', but expected '%v'", s3, expect3)) - } - */ -} - -func reportTestBadgersNeedingClose() error { - globalBadgerReg.mu.Lock() - defer globalBadgerReg.mu.Unlock() - n := len(globalBadgerReg.mp) - if n == 0 { - return nil - } - AlwaysPrintf("*** these badgers are still open (n=%v):", n) - i := 0 - for w := range globalBadgerReg.mp { - AlwaysPrintf("i=%v, w p=%p stack:\n%v\n\n", i, w, w.startStack) - i++ - } - return errors.New("unclosed badgers, contact Animal Control") -} - -/* -func TestBadger_ConflictWriteWriteResolution(t *testing.T) { - - // 1) when do we get write-write conflicts (probably different goroutines) but - // can we get them on different keys? - - // 2) does having a lock registry that insures we are only ever writing - // different keys at once avoid write-write conflicts? - - // 3) how should write-write conflicts be resolved? - // presumably just retying the write? - - // setup - dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ConflictWriteWriteResolution") - defer clean() - defer dbwrap.Close() - - concur := 10 - //bkey := []byte("a") - //by := []byte("value-for-a") - - // read-loop: - for i := 0; i < concur*2; i++ { - go func() { - tx, _ := dbwrap.NewTx(!writable, "", Txo{}) - - for j := 0; true; j++ { - - bkey := []byte(fmt.Sprintf("key-for-a j=%v", j)) - //by := []byte(fmt.Sprintf("value-for-a j=%v", -1)) - - _, err := tx.tx.Get(bkey) - if err != badger.ErrKeyNotFound { - panicOn(err) - } - - if j%10 == 0 { - vv("committing after 10, gid=%v", curGID()) - tx.Rollback() - tx, _ = dbwrap.NewTx(!writable, "", Txo{}) - } - - } - }() - } - - // write-loop: - for i := 0; i < concur; i++ { - go func() { - tx, _ := dbwrap.NewTx(writable, "", Txo{}) - - for j := 0; true; j++ { - - bkey := []byte(fmt.Sprintf("key-for-a j=%v", j)) - by := []byte(fmt.Sprintf("value-for-a j=%v", j)) - - entry := badger.NewEntry(bkey, by) - err := tx.tx.SetEntry(entry) - panicOn(err) - - if j%5 == 0 { - panicOn(tx.tx.Delete(bkey)) - } - - _, err = tx.tx.Get(bkey) - if err != badger.ErrKeyNotFound { - panicOn(err) - } - - if j%10 == 0 { - ///vv("committing after 10, gid=%v", curGID()) - err = tx.tx.Commit() - panicOn(err) - tx, _ = dbwrap.NewTx(writable, "", Txo{}) - } - - } - }() - } - select {} -} -*/ diff --git a/dbshard.go b/dbshard.go index 27bc4c6f9..0c578b513 100644 --- a/dbshard.go +++ b/dbshard.go @@ -376,7 +376,7 @@ func (dbs *DBShard) DumpAll() { case roaringTxn: case rbfTxn: case lmdbTxn: - case badgerTxn: + case boltTxn: default: panic(fmt.Sprintf("unknown txtyp: '%v'", ty)) } @@ -410,7 +410,13 @@ func (dbs *DBShard) pathForType(ty txtype) string { // what here for roaring? well, roaringRegistrar.OpenDBWrapper() // is a no-op anyhow. so doesn't need to be correct atm. - return dbs.HolderPath + sep + dbs.Index + ".index.txstores@@@" + sep + "store" + ty.FileSuffix() + "@" + sep + fmt.Sprintf("shard.%04v%v", dbs.Shard, ty.FileSuffix()) + path := dbs.HolderPath + sep + dbs.Index + ".index.txstores@@@" + sep + "store" + ty.FileSuffix() + "@" + sep + fmt.Sprintf("shard.%04v%v", dbs.Shard, ty.FileSuffix()) + if ty == boltTxn { + // special case: + // bolt doesn't use a directory like the others, just a direct path. + path += sep + "bolt.db" + } + return path } // if you don't know the shard, you have to use this. @@ -466,8 +472,8 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs * registry = globalRbfDBReg case lmdbTxn: registry = globalLMDBReg - case badgerTxn: - registry = globalBadgerReg + case boltTxn: + registry = globalBoltReg default: panic(fmt.Sprintf("unknown txtyp: '%v'", ty)) } diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go index ac68a4fd4..7cbcb95da 100644 --- a/dbshard_internal_test.go +++ b/dbshard_internal_test.go @@ -74,7 +74,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { orig := os.Getenv("PILOSA_TXSRC") defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests! - for _, src := range []string{"lmdb", "roaring", "badger", "rbf"} { + for _, src := range []string{"lmdb", "roaring", "bolt", "rbf"} { os.Setenv("PILOSA_TXSRC", src) @@ -149,13 +149,13 @@ rick.index.txstores@@@/store-lmdb@@/shard.0219-lmdb@ rick.index.txstores@@@/store-lmdb@@/shard.0221-lmdb@ rick.index.txstores@@@/store-lmdb@@/shard.0223-lmdb@ `, - "badger": ` -rick.index.txstores@@@/store-badgerdb@@/shard.0093-badgerdb@ -rick.index.txstores@@@/store-badgerdb@@/shard.0215-badgerdb@ -rick.index.txstores@@@/store-badgerdb@@/shard.0217-badgerdb@ -rick.index.txstores@@@/store-badgerdb@@/shard.0219-badgerdb@ -rick.index.txstores@@@/store-badgerdb@@/shard.0221-badgerdb@ -rick.index.txstores@@@/store-badgerdb@@/shard.0223-badgerdb@ + "bolt": ` +rick.index.txstores@@@/store-boltdb@@/shard.0093-boltdb@/bolt.db +rick.index.txstores@@@/store-boltdb@@/shard.0215-boltdb@/bolt.db +rick.index.txstores@@@/store-boltdb@@/shard.0217-boltdb@/bolt.db +rick.index.txstores@@@/store-boltdb@@/shard.0219-boltdb@/bolt.db +rick.index.txstores@@@/store-boltdb@@/shard.0221-boltdb@/bolt.db +rick.index.txstores@@@/store-boltdb@@/shard.0223-boltdb@/bolt.db `, "rbf": ` rick.index.txstores@@@/store-rbfdb@@/shard.0093-rbfdb@ @@ -189,8 +189,8 @@ func makeSampleRoaringDir(root, txsrc string, minBytes int, h *Holder) { // DBPerShard won't know anything about it. helperCreateDBShard(h, index, shard) continue - case "badger": - makeBadgertestDB(root+sep+fn, h, shard) + case "bolt": + makeBolttestDB(root+sep+fn, h, shard) helperCreateDBShard(h, index, shard) continue case "rbf": @@ -227,10 +227,10 @@ func makeLMDBtestDB(path string, h *Holder, shard uint64) { } -func makeBadgertestDB(path string, h *Holder, shard uint64) { +func makeBolttestDB(path string, h *Holder, shard uint64) { i := uint64(1) - w, _ := mustOpenEmptyBadgerWrapper(path) - badgerDBMustSetBitvalue(w, "index", "field", "view", shard, i) + w, _ := mustOpenEmptyBoltWrapper(path) + BoltMustSetBitvalue(w, "index", "field", "view", shard, i) w.Close() } diff --git a/dbshard_test.go b/dbshard_test.go index fb6d7e00f..9912e03d9 100644 --- a/dbshard_test.go +++ b/dbshard_test.go @@ -35,15 +35,7 @@ func skipForNonLMDB(t *testing.T) { } } -var _ = skipForNonLMDB // happy linter -var _ = skipForNonBadger // happy linter - -func skipForNonBadger(t *testing.T) { - src := os.Getenv("PILOSA_TXSRC") - if src != "badger" { - t.Skip("skip if not badger") - } -} +var _ = skipForNonLMDB // happy linter // Can't write it all to one shard like we do (did). func Test_DBPerShard_multiple_shards_used(t *testing.T) { diff --git a/executor.go b/executor.go index 38e05911c..08d0feb47 100644 --- a/executor.go +++ b/executor.go @@ -3138,7 +3138,6 @@ func (e *executor) executeRowShard(ctx context.Context, qcx *Qcx, index string, // Handle bsiGroup ranges differently. if c.HasConditionArg() { - // looks the same on badger/roaring. we think. return e.executeRowBSIGroupShard(ctx, qcx, index, c, shard) } diff --git a/field_internal_test.go b/field_internal_test.go index 3423cc432..deea358e4 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -914,6 +914,6 @@ func TestBSIGroup_TxReopenDB(t *testing.T) { panicOn(qcx.Finish()) } // loop - // the test: can we re-open a BSI fragment under badger/rbf. + // the test: can we re-open a BSI fragment under Tx store _ = f.Reopen() } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 8514f7284..9e5a567bc 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -560,7 +560,7 @@ func TestFragment_Sum(t *testing.T) { } }) - panicOn(tx.Commit()) + tx.Rollback() tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() @@ -2178,7 +2178,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) { } } - panicOn(tx.Commit()) + tx.Rollback() tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() @@ -2457,7 +2457,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) { } } - panicOn(tx.Commit()) + tx.Rollback() tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() @@ -2707,7 +2707,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) { } } - panicOn(tx.Commit()) + tx.Rollback() tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() @@ -2787,7 +2787,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { if err := f.bulkImport(tx, rows, cols, options); err != nil { b.Fatalf("Error Building Sample: %s", err) } - panicOn(tx.Commit()) + tx.Rollback() if row > max { max = row } @@ -2949,7 +2949,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { // the cost of actually doing the op log for the large initial data set // is excessive. force storage into snapshotted state, then use import // to generate an op log and/or snapshot. - // note: skipped for badger, above. + // note: skipped for rbf, bolt, lmdb, above. _, _, err := frags[j].storage.ImportRoaringBits(data, false, false, 0) if err != nil { b.Fatalf("importing roaring: %v", err) @@ -3376,7 +3376,7 @@ func BenchmarkFileWrite(b *testing.B) { ///////////////////////////////////////////////////////////////////// -// not called under badger b/c f.idx.NeedsSnapshot() in Clean() avoids it. +// not called under Tx stores b/c f.idx.NeedsSnapshot() in Clean() avoids it. func (f *fragment) sanityCheck(t testing.TB) { newBM := roaring.NewFileBitmap() file, err := os.Open(f.path) @@ -3393,7 +3393,7 @@ func (f *fragment) sanityCheck(t testing.TB) { t.Fatalf("sanityCheck couldn't unmarshal fragment %s: %v", f.path, err) } // Refactor fragment.storage - // note: not called for badger, see above. + // note: not called for rbf, see above. if equal, reason := newBM.BitwiseEqual(f.storage); !equal { t.Fatalf("fragment %s: unmarshalled bitmap different: %v", f.path, reason) } @@ -3408,11 +3408,11 @@ func (f *fragment) Clean(t testing.TB) { // check or else, in some cases, the background snapshot queue // can decide to pick it up. func() { - // should we skip snapshot queue stuff under badger/rbf? + // should we skip snapshot queue stuff under bolt/rbf? defer f.mu.Unlock() - // badger doesn't need snapshot, so this stuff is skipped. - // The snapshot queue stuff doesn't work under badger. + // rbf doesn't need snapshot, so this stuff is skipped. + // The snapshot queue stuff doesn't work under rbf. if f.idx.NeedsSnapshot() { err := f.holder.SnapshotQueue.Await(f) if err != nil { diff --git a/go.mod b/go.mod index 2c94909b2..54169033d 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect - github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 github.com/dustin/go-humanize v1.0.0 github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 github.com/glycerine/lmdb-go v1.9.34 @@ -36,6 +35,7 @@ require ( github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v2.18.12+incompatible github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.4.0 @@ -50,6 +50,7 @@ require ( golang.org/x/text v0.3.3 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/grpc v1.28.0 + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible diff --git a/go.sum b/go.sum index 0389c9b4b..f160e1afe 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,6 @@ github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWu github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= -github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= @@ -33,24 +31,16 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= -github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 h1:JBNM90aGLCiF9iJYvpvayMpYeW498v5ZDZqE2chqZ2A= -github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -88,8 +78,6 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= @@ -208,7 +196,6 @@ github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qq github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -231,7 +218,6 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= @@ -240,7 +226,6 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -258,7 +243,6 @@ github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0= @@ -278,7 +262,6 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -320,11 +303,9 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/holder_test.go b/holder_test.go index 7ccad46f1..0ff7aa272 100644 --- a/holder_test.go +++ b/holder_test.go @@ -808,7 +808,7 @@ func TestHolderSyncer_IntField(t *testing.T) { t.Fatalf("syncing node 1: %v", err) } - // dump the badger keys for both c0 and c1 + // dump the rbf keys for both c0 and c1 // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { diff --git a/lmdb.go b/lmdb.go index d4251383c..dbe539897 100644 --- a/lmdb.go +++ b/lmdb.go @@ -39,6 +39,8 @@ import ( "github.com/pkg/errors" ) +const TxInitialMmapSize = 4 << 30 // 4GB + // Linux builds note: // // This setting of -DMDB_USE_SYSV_SEM=1 is required in the cgo build @@ -160,7 +162,7 @@ func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper err = env.SetMaxDBs(1) panicOn(err) //err = env.SetMapSize(256 << 30) // 256GB - err = env.SetMapSize(4 << 30) // 4GB + err = env.SetMapSize(TxInitialMmapSize) panicOn(err) panicOn(os.MkdirAll(filepath.Dir(path), 0755)) @@ -563,8 +565,8 @@ const isDebugRun = false // Rollback rolls back the transaction. func (tx *LMDBTx) Rollback() { - alreadyDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) - if !alreadyDone { + notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) + if !notDone { return } //vv("lmdb rollback tx _sn_ %v; stack \n%v", tx.sn) // , stack()) @@ -590,8 +592,8 @@ func (tx *LMDBTx) Rollback() { // Commits can handle up to 100k updates to fragments // at once, but not more. This is a LMDBDB imposed limit. func (tx *LMDBTx) Commit() error { - alreadyDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) - if !alreadyDone { + notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) + if !notDone { return nil } //vv("lmdb commit tx _sn_ %v; stack \n%v", tx.sn, stack()) @@ -892,7 +894,7 @@ type LMDBIterator struct { } // NewLMDBIterator creates an iterator on tx that will -// only return badgerKeys that start with prefix. +// only return rbf that start with prefix. func NewLMDBIterator(tx *LMDBTx, prefix []byte) (bi *LMDBIterator) { tx.mu.Lock() diff --git a/lmdb_other.go b/lmdb_other.go index 955ea66d6..fa6abb946 100644 --- a/lmdb_other.go +++ b/lmdb_other.go @@ -31,6 +31,13 @@ import ( var _ = time.Now +const isDebugRun = false +const TxInitialMmapSize = 1 << 30 // 1GB + +func ToContainer(typ byte, w []byte) (c *roaring.Container) { + panic("ToContainer not implemented yet on non-amd64") +} + // lmdbRegistrar facilitates shutdown // of all the lmdb databases started under // tests. Its needed because most tests don't cleanup @@ -252,7 +259,7 @@ func (tx *LMDBTx) ContainerIterator(index, field, view string, shard uint64, fir type LMDBIterator struct{} // NewLMDBIterator creates an iterator on tx that will -// only return badgerKeys that start with prefix. +// only return lmdbKeys that start with prefix. func NewLMDBIterator(tx *LMDBTx, prefix []byte) (bi *LMDBIterator) { panic("lmdb only available on 64-bit arch") } diff --git a/lmdb_test.go b/lmdb_test.go index 2b69a9927..69103f8ef 100644 --- a/lmdb_test.go +++ b/lmdb_test.go @@ -932,12 +932,6 @@ func TestLMDB_RoaringBitmap(t *testing.T) { } } -// no reverse iterator on LMDB; we did a special case for Max -// rather than a general purpose reverse iterator which we -// aren't using for anything else. -//func TestLMDB_reverse_badger_iterator_and_prefix_valid(t *testing.T) -//func TestLMDB_just_reverse_badger_iterator_and_prefix_valid(t *testing.T) - func TestLMDB_ImportRoaringBits(t *testing.T) { dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_ImportRoaringBits") @@ -946,7 +940,7 @@ func TestLMDB_ImportRoaringBits(t *testing.T) { index, field, view, shard := "i", "f", "v", uint64(0) tx, _ := dbwrap.NewTx(writable, index, Txo{}) defer tx.Rollback() - tx.(*LMDBTx).DeleteEmptyContainer = true // traditional badger Tx behavior, but not Roaring. + tx.(*LMDBTx).DeleteEmptyContainer = true // match Roaring //bitvalue := uint64(42) diff --git a/rrtx.go b/rrtx.go index 4868e443e..8e254496e 100644 --- a/rrtx.go +++ b/rrtx.go @@ -575,7 +575,7 @@ func (w *RoaringWrapper) DeleteField(index, field, fieldPath string) error { // match txn sn count vs lmdb/etc. atomic.AddInt64(&globalNextTxSnRoaring, 1) - // under blue-green badger_roaring, the directory will not be found, b/c badger will have + // under blue-green bolt_roaring, the directory will not be found, b/c bolt will have // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: // "If the path does not exist, RemoveAll returns nil (no error)" err := os.RemoveAll(fieldPath) diff --git a/tournament.sh b/tournament.sh index 31c3d42c3..af78ec798 100755 --- a/tournament.sh +++ b/tournament.sh @@ -5,7 +5,7 @@ ## one or two backends through the rigors of make testv-race. ## logs are saved to the tourna.log.${i} files. -for i in rbf lmdb roaring rbf_lmdb rbf_roaring lmdb_rbf lmdb_roaring roaring_rbf roaring_lmdb ; do +for i in rbf lmdb roaring bolt rbf_lmdb rbf_roaring lmdb_rbf lmdb_roaring roaring_rbf roaring_lmdb roaring_bolt lmdb_bolt; do echo "$(date) starting ${i}, output to tourna.log.${i}" echo "***=== ${i} ====================*** $(date)" &> tourna.log.${i} PILOSA_TXSRC=${i} make testv-race 2>&1 > tourna.log.${i} diff --git a/tx.go b/tx.go index c6bc1066b..61c2dc212 100644 --- a/tx.go +++ b/tx.go @@ -48,7 +48,7 @@ const writable = true // that have not been committed. type Tx interface { - // Type returns "roaring", "rbf", "badger", "badger_roaring", or one of the other + // Type returns "roaring", "rbf", "bolt", "badger_roaring", or one of the other // blue-green Tx types at the top of txfactory.go Type() string diff --git a/txfactory.go b/txfactory.go index 9f01e845a..c04889500 100644 --- a/txfactory.go +++ b/txfactory.go @@ -38,7 +38,7 @@ const ( RoaringTxn string = "roaring" LmdbTxn string = "lmdb" RBFTxn string = "rbf" - BadgerTxn string = "badger" + BoltTxn string = "bolt" ) // DefaultTxsrc is set here. pilosa/server/config.go references it @@ -409,12 +409,12 @@ const ( roaringTxn txtype = 1 // these don't really have any transactions rbfTxn txtype = 2 lmdbTxn txtype = 3 - badgerTxn txtype = 4 + boltTxn txtype = 4 ) // these need to be skipped by the holder.go field scanner that // calls IsTxDatabasePath -var allTypesWithSuffixes = []txtype{rbfTxn, lmdbTxn, badgerTxn} +var allTypesWithSuffixes = []txtype{rbfTxn, lmdbTxn, boltTxn} // FileSuffix is used to determine backend directory names. // We append '@' to be sure we never collide with a field name @@ -429,8 +429,8 @@ func (ty txtype) FileSuffix() string { return "-rbfdb@" case lmdbTxn: return "-lmdb@" - case badgerTxn: - return "-badgerdb@" + case boltTxn: + return "-boltdb@" } panic(fmt.Sprintf("unkown txtype %v", int(ty))) } @@ -478,8 +478,8 @@ func MustTxsrcToTxtype(txsrc string) (types []txtype) { types = append(types, rbfTxn) case LmdbTxn: // "lmdb" types = append(types, lmdbTxn) - case BadgerTxn: // "badger" - types = append(types, badgerTxn) + case BoltTxn: // "bolt" + types = append(types, boltTxn) default: panic(fmt.Sprintf("unknown txsrc '%v'", s)) } @@ -844,8 +844,8 @@ func (ty txtype) String() string { return "rbfTxn" case lmdbTxn: return "lmdbTxn" - case badgerTxn: - return "badgerTxn" + case boltTxn: + return "boltTxn" } panic(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty))) } @@ -1212,7 +1212,7 @@ func anyGlobalDBWrappersStillOpen() bool { if globalLMDBReg.Size() != 0 { return true } - if globalBadgerReg.Size() != 0 { + if globalBoltReg.Size() != 0 { return true } return false diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go index d172099d5..0db9a5a44 100644 --- a/txfactory_internal_test.go +++ b/txfactory_internal_test.go @@ -26,10 +26,10 @@ import ( func Test_TxFactory_Qcx_query_context(t *testing.T) { src := os.Getenv("PILOSA_TXSRC") - if src == "rbf" || src == "lmdb" || src == "badger" { + if src == "rbf" || src == "lmdb" || src == "bolt" { // ok } else { - t.Skip("this test only for lmdb and rbf and badger") + t.Skip("this test only for lmdb and rbf and bolt") } shard := uint64(0) @@ -251,7 +251,7 @@ func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) { orig := os.Getenv("PILOSA_TXSRC") defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests! - checked := []string{"lmdb", "roaring", "badger", "rbf"} + checked := []string{"lmdb", "roaring", "bolt", "rbf"} for _, blue := range checked { for _, green := range checked { diff --git a/utils_internal_test.go b/utils_internal_test.go index 084389532..42865c3dc 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -497,7 +497,7 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error // this is the *test* version of a network call, transferring fragments between // nodes in a cluster. So it is allowed to be kind of a hack. - // there will be two -badgerdb directories/databases, we need to copy + // there will be two -rbfdb directories/databases, we need to copy // from src to dest the fragment. This simulates sending the fragment over the network. srcIdx := srcCluster.holder.Index(src.Index) srctx := srcIdx.holder.txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment, Shard: srcFragment.shard})