diff --git a/.circleci/config.yml b/.circleci/config.yml index 42de1db0c..6ecff4418 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,6 +55,13 @@ jobs: - checkout-plus - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.23.8 - run: make golangci-lint + go-mod-tidy: + executor: + name: golang + steps: + - checkout-plus + - run: go mod tidy + - run: git diff --exit-code -- go.mod go.sum test-build-arm: executor: name: golang @@ -168,6 +175,9 @@ workflows: - check-license-headers: requires: - setup + - go-mod-tidy: + requires: + - setup - test-build-arm: requires: - setup diff --git a/Makefile b/Makefile index e1577b62b..e38ed3860 100644 --- a/Makefile +++ b/Makefile @@ -16,16 +16,13 @@ RELEASE_ENABLED = $(subst 0,,$(RELEASE)) NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" ) BUILD_TAGS += $(if $(RELEASE_ENABLED),release) BUILD_TAGS += shardwidth$(SHARD_WIDTH) -BUILD_TAGS += $(foreach p,$(PLUGINS),plugin$(p)) define LICENSE_HASH_CODE head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " " endef LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go)) -PLUGINS=distinct export GO111MODULE=on export GOPRIVATE=github.com/molecula -export PLUGINS # Run tests and compile Pilosa default: test build @@ -97,7 +94,7 @@ clustertests: vendor # Like clustertests, but rebuilds all images. clustertests-build: vendor - docker-compose -f $(DOCKER_COMPOSE) down + docker-compose -f $(DOCKER_COMPOSE) down -v docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build # Create prerelease builds @@ -152,25 +149,76 @@ docker-test: # run top tests, not subdirs. print summary red/green after. # The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt. topt: + mv log.topt.roar log.topt.roar.prev || true go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.roar @echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l @echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l topt-badger: + mv log.topt.badger log.topt.badger.prev || true PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger @echo " log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l @echo " log.topt.badger red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l +topt-badger-race: + mv log.topt.badger-race log.topt.badger-race.prev || true + PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger-race + @echo " log.topt.badger-race green: \c"; cat log.topt.badger-race | grep PASS |wc -l + @echo " log.topt.badger-race red: \c"; cat log.topt.badger-race | grep '\-\-\- FAIL' |wc -l + topt-rbf: + mv log.topt.rbf log.topt.rbf.prev || true PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.rbf @echo " log.topt.rbf green: \c"; cat log.topt.rbf | grep PASS |wc -l @echo " log.topt.rbf red: \c"; cat log.topt.rbf | grep '\-\-\- FAIL' |wc -l topt-race: + mv log.topt.race log.topt.race.prev || true go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.race @echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l @echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l +# blue-green checks. These run two different storage engines (rbf, roaring, or badger) +# and compare each transaction for a result. + +bg-rr: # shorthand for bluegreen test with A:badger; B:roaring + mv log.bg-rr log.bg-rr.prev || true + set -o pipefail; PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rr + @echo " log.bg-rr green: \c"; cat log.bg-rr | grep PASS |wc -l + @echo " log.bg-rr red: \c"; cat log.bg-rr | grep '\-\-\- FAIL' |wc -l + +rr-bg: # bluegreen with A:roaring; B:badger (B's values are returned). + mv log.bg.roar_bg log.bg.roar_bg.prev || true + set -o pipefail; PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-bg + ##PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-bg + @echo " log.rr-bg green: \c"; cat log.rr-bg | grep PASS |wc -l + @echo " log.rr-bg red: \c"; cat log.rr-bg | grep '\-\-\- FAIL' |wc -l + +rbf-rr: + mv log.rbf-rr log.rbf-rr.prev || true + set -o pipefail; PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-rr + @echo " log.rbf-rr green: \c"; cat log.rbf-rr | grep PASS |wc -l + @echo " log.rbf-rr red: \c"; cat log.rbf-rr | grep '\-\-\- FAIL' |wc -l + +rr-rbf: + mv log.rr-rbf log.rr-rbf.prev || true + set -o pipefail; PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-rbf + @echo " log.rr-rbf green: \c"; cat log.rr-rbf | grep PASS |wc -l + @echo " log.rr-rbf red: \c"; cat log.rr-rbf | grep '\-\-\- FAIL' |wc -l + +rbf-bg: + mv log.rbf-bg log.rbf-bg.prev || true + set -o pipefail; PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-bg + @echo " log.rbf-bg green: \c"; cat log.rbf-bg | grep PASS |wc -l + @echo " log.rbf-bg red: \c"; cat log.rbf-bg | grep '\-\-\- FAIL' |wc -l + +bg-rbf: + mv log.bg-rbf log.bg-rbf.prev || true + set -o pipefail; PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rbf + @echo " log.bg-rbf green: \c"; cat log.bg-rbf | grep PASS |wc -l + @echo " log.bg-rbf red: \c"; cat log.bg-rbf | grep '\-\-\- FAIL' |wc -l + + # Run golangci-lint golangci-lint: require-golangci-lint golangci-lint run --skip-files '.*\.peg\.go' diff --git a/api.go b/api.go index b5bc89422..3ed7358e3 100644 --- a/api.go +++ b/api.go @@ -371,16 +371,9 @@ func importWorker(importWork chan importJob) { var doClear bool switch doAction { case RequestActionOverwrite: - // TODO(jea): the question here is, why are we commiting this separately from j.tx? - // why doesn't j.tx suffice? It doesn't but why/which is correct? - tx := j.field.holder.indexes[j.field.index].Txf.NewTx(Txo{Write: true, Field: j.field}) - defer tx.Rollback() - if err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block); err != nil { + if err := j.field.importRoaringOverwrite(j.ctx, j.tx, viewData, j.shard, viewName, j.req.Block); err != nil { return errors.Wrap(err, "importing roaring as overwrite") } - if err := tx.Commit(); err != nil { - return errors.Wrap(err, "commit of importing roaring as overwrite") - } case RequestActionClear: doClear = true fallthrough diff --git a/api_test.go b/api_test.go index 3942a7337..b52906514 100644 --- a/api_test.go +++ b/api_test.go @@ -165,6 +165,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) { } func TestAPI_Import(t *testing.T) { + skipForRBF(t) + c := test.MustRunCluster(t, 2, []server.CommandOption{ server.OptCommandServerOptions( @@ -276,6 +278,8 @@ func TestAPI_Import(t *testing.T) { } func TestAPI_ImportValue(t *testing.T) { + skipForRBF(t) + c := test.MustRunCluster(t, 2, []server.CommandOption{ server.OptCommandServerOptions( diff --git a/badger.go b/badger.go index 65052b525..bcd95f8da 100644 --- a/badger.go +++ b/badger.go @@ -15,8 +15,9 @@ package pilosa import ( - "errors" + "bytes" "fmt" + "io" "io/ioutil" "log" "os" @@ -29,7 +30,9 @@ import ( "unsafe" badger "github.com/dgraph-io/badger/v2" + badgeroptions "github.com/dgraph-io/badger/v2/options" "github.com/pilosa/pilosa/v2/roaring" + "github.com/pkg/errors" ) // TODO: is there a more optimal time to do badger garbage collection? @@ -189,15 +192,70 @@ func (l *BadgerLog) Debugf(f string, v ...interface{}) { l.Printf("DEBUG: "+f, v...) } +// badgerRegistrar facilitates shutdown +// of all the badger databases started under +// tests. Its needed because most tests don't cleanup +// the *Index(es) they create. But we still +// want to shutdown badgerDB goroutines +// after tests run. +// +// It also allows opening the same path twice to +// result in sharing the same open database handle, and +// thus the same transactional guarantees. +// +type badgerRegistrar struct { + mu sync.Mutex + mp map[*BadgerDBWrapper]bool + + path2db map[string]*BadgerDBWrapper +} + +var globalBadgerReg *badgerRegistrar = newBadgerTestRegistrar() + +func newBadgerTestRegistrar() *badgerRegistrar { + return &badgerRegistrar{ + mp: make(map[*BadgerDBWrapper]bool), + path2db: make(map[string]*BadgerDBWrapper), + } +} + +// register each badger created under tests, so we +// can clean them up. This is called by openBadgerDBWrapper() while +// holding the r.mu.Lock, since it needs to atomically +// check the registry and make a new instance only +// if one does not exist for its path, and otherwise +// return the existing instance. +func (r *badgerRegistrar) unprotectedRegister(w *BadgerDBWrapper) { + r.mp[w] = true + r.path2db[w.path] = w +} + +// unregister removes w from r +func (r *badgerRegistrar) unregister(w *BadgerDBWrapper) { + r.mu.Lock() + delete(r.mp, w) + delete(r.path2db, w.path) + r.mu.Unlock() +} + +func DumpAllBadger() { + globalBadgerReg.mu.Lock() + defer globalBadgerReg.mu.Unlock() + for w := range globalBadgerReg.mp { + _ = w + AlwaysPrintf("this badger path='%v' has: \n%v\n", w.path, w.StringifiedBadgerKeys(nil)) + } +} + // newBadgerDBWrapper creates a new empty database, blowing away // any prior path + "-badgerdb" directory. -func newBadgerDBWrapper(path string) (*BadgerDBWrapper, error) { +func (r *badgerRegistrar) newBadgerDBWrapper(path string) (*BadgerDBWrapper, error) { bpath := badgerPath(path) err := os.RemoveAll(bpath) if err != nil { return nil, err } - return openBadgerDBWrapper(bpath) + return r.openBadgerDBWrapper(bpath) } // badgerPath is a helper for determining the full directory @@ -212,17 +270,46 @@ func badgerPath(path string) string { // openBadgerDB opens the database in the bpath directoy // without deleting any prior content. Any BadgerDB // database directory will have the "-badgerdb" suffix. -func openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) { - +// +// openBadgerDB will check the registry and make a new instance only +// if one does not exist for its bpath. Otherwise it returns +// the existing instance. This insures only one badgerDB +// per bpath in this pilosa node. +func (r *badgerRegistrar) openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) { // now that newTxFactory can call us directly, we might not // have the -badgerdb suffix. if !strings.HasSuffix(bpath, "-badgerdb") { bpath += "-badgerdb" } + r.mu.Lock() + defer r.mu.Unlock() + w, ok := r.path2db[bpath] + if ok { + // creates the effect of having only one badger open per pilosa node. + return w, nil + } + // otherwise, make a new badger and store it in globalBadgerReg + // regular: works on amd64, but 386 doesn't work. opt := badger.DefaultOptions(bpath).WithLogger(badgerDefaultLogger) + opt.Compression = badgeroptions.None // turn off compression. + opt.ZSTDCompressionLevel = 0 // really, just in case. + opt.SyncWrites = true // default is true, safe. + + // MaxCacheSize docs: + // + // how much data cache should hold in memory. A small size of + // cache means lower memory consumption and lookups/iterations + // would take longer. It is recommended to use a cache if you're + // using compression or encryption. If compression and + // encryption both are disabled, adding a cache will lead to + // unnecessary overhead which will affect the read performance. + // Setting size to zero disables the cache altogether. + opt.MaxCacheSize = 0 + opt.LoadBloomsOnOpen = false // should speed up start-up time. + // to get memory only do: //opt := badger.DefaultOptions("").WithLogger(badgerDefaultLogger).WithInMemory(true) @@ -231,12 +318,16 @@ func openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) { return nil, err } halt := make(chan bool) - w := &BadgerDBWrapper{ + w = &BadgerDBWrapper{ + reg: r, path: bpath, db: db, halt: halt, hasher: NewBlake3Hasher(), } + r.unprotectedRegister(w) + + w.startStack = stack() w.startBadgerGarbageCollectionBackgroundGoro() return w, nil } @@ -251,77 +342,8 @@ func (w *BadgerDBWrapper) DeleteIndex(indexName string) error { if strings.Contains(indexName, "'") { return fmt.Errorf("error: bad indexName `%v` in BadgerDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes.", indexName) } - w.muDb.Lock() - defer w.muDb.Unlock() - - // a) do key-ony iteration, no value fetch; - // - // b) do deletes in large batches, to avoid alot of txn overhead; - // per recommendation https://github.com/dgraph-io/badger/issues/598 - // - // c) we do not, at present, try to maintain one large - // transaction with all the keys in a index in it. Because - // there can be too many keys. Hence the index will disappear - // in chucks of 100K keys, not atomically-all-at-once. - prefix := badgerIndexOnlyPrefix(indexName) - - noMoreKeysWithPrefix := false - const maxDeletesPerTxn = 100000 - - for !noMoreKeysWithPrefix { - err := w.db.Update(func(txn *badger.Txn) error { - o := badger.DefaultIteratorOptions - o.AllVersions = false - o.PrefetchValues = false // key-only iteration, no values. - - // note: panic: Unclosed iterator at time of Txn.Discard ? panic on segfault here? - // This means we messed up and Closed() the Database already; too early. - it := txn.NewIterator(o) - - defer it.Close() - n := 0 - goners := make([][]byte, 0, maxDeletesPerTxn) - for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { - - // KeyCopy() is required; Key() means corruption and possible segfault. - key := it.Item().KeyCopy(nil) - goners = append(goners, key) - n++ - if n >= maxDeletesPerTxn { - break - } - } - if !it.ValidForPrefix(prefix) { - noMoreKeysWithPrefix = true // done with the full delete of up to maxDeletesPerTxn - } - for _, key := range goners { - if err := txn.Delete(key); err != nil { - return err - } - } - return nil // auto-commit happens - }) - // err back from Update can be ErrConflict in case of - // a conflict. Badger docs: "Depending on the state - // of your application, you have the option to - // retry the operation if you receive this error." - panicOn(err) - - } // end for: proceed to next bath of 100K keys - - // Finally, run a garbage collection to delete values from the value log. - // - // "Only one GC is allowed at a time. If another value log GC - // is running, or DB has been closed, this would return an ErrRejected." - // -- https://godoc.org/github.com/dgraph-io/badger#DB.RunValueLogGC - // Still, we don't see a mutex inside the RunValueLogGC code, so - // lock muGC just to be sure. - w.muGC.Lock() - defer w.muGC.Unlock() - _ = w.db.RunValueLogGC(0.5) - - return nil + return w.DeletePrefix(prefix) } // startBadgerGarbageCollectionBackgroundGoro handles Badger DB @@ -367,6 +389,9 @@ type BadgerDBWrapper struct { path string db *badger.DB + // track our registrar for Close / goro leak reporting purposes. + reg *badgerRegistrar + // openTx and openIt are BadgerDBWrapper scoped tables of all open // transactions and iterators. These are primarily for debugging purposes. // openTx and openIt should only be read/written after locking the muOpenTxIt mutex. @@ -404,6 +429,12 @@ type BadgerDBWrapper struct { // safety because otherwise TestAPI_ImportColumnAttrs sees // corrupted data. doAllocZero bool + + // stack() from our creation point, to track tests + // that haven't closed us. + startStack string + + DeleteEmptyContainer bool } // unprotectedListOpenTxAsString is a debugging helper. @@ -437,24 +468,31 @@ func (w *BadgerDBWrapper) UnprotectedListOpenItAsString() (r string) { // Read-only queries should set write to false, to allow more concurrency. // Methods on a BadgerTx are thread-safe, and can be called from // different goroutines. -func (w *BadgerDBWrapper) NewBadgerTx(write bool) (tx *BadgerTx) { +// +// initialIndexName is optional. It is set by the TxFactory from the Txo +// options provided at the Tx creation point. It allows us to recognize +// and isolate cross-index queries more quickly. It can always be empty "" +// but when set is highly useful for debugging. It has no impact +// on transaction behavior. +// +func (w *BadgerDBWrapper) NewBadgerTx(write bool, initialIndexName string) (tx *BadgerTx) { w.muDb.Lock() defer w.muDb.Unlock() tx = &BadgerTx{ - write: write, - tx: w.db.NewTransaction(write), - Db: w, - initloc: stack(), - doAllocZero: w.doAllocZero, + write: write, + tx: w.db.NewTransaction(write), + Db: w, + initloc: stack(), + doAllocZero: w.doAllocZero, + initialIndexName: initialIndexName, + DeleteEmptyContainer: w.DeleteEmptyContainer, } - //vv("NewBadgerTx(write=%v) top, p=%p", write, tx) - //pp("NewBadgerTx(write=%v) top, p=%p, stack=\n\n'%v'", write, tx, stack()) if w.openTx == nil { w.openTx = make(map[*BadgerTx]bool) } - //pp("NewBadgerTx(write=%v); p=%p; (currently open txn: '%v', its: '%v'). initloc:'%v'", write, tx, w.unprotectedListOpenTxAsString(), w.UnprotectedListOpenItAsString(), tx.initloc) + w.muOpenTxIt.Lock() w.openTx[tx] = write w.muOpenTxIt.Unlock() @@ -466,6 +504,7 @@ func (w *BadgerDBWrapper) Close() (err error) { w.muDb.Lock() defer w.muDb.Unlock() if !w.closed { + w.reg.unregister(w) close(w.halt) w.closed = true } @@ -501,8 +540,17 @@ type BadgerTx struct { // for tracking txn boundary issues, track all the memory // that we deploy for roaring containers, and zero it on // transaction commit/rollback. + acMu sync.Mutex // protect ourAllocs and ourContainers ourAllocs [][]byte ourContainers []*roaring.Container + + initialIndexName string + + DeleteEmptyContainer bool +} + +func (tx *BadgerTx) Type() string { + return BadgerTxn } func (tx *BadgerTx) UseRowCache() bool { @@ -521,6 +569,8 @@ func (tx *BadgerTx) UseRowCache() bool { // to transaction commit. func (tx *BadgerTx) overWriteOurAllocs() { + tx.acMu.Lock() + defer tx.acMu.Unlock() for _, s := range tx.ourAllocs { // The Go compiler recognizes the following pattern and inserts @@ -529,6 +579,10 @@ func (tx *BadgerTx) overWriteOurAllocs() { // and https://codereview.appspot.com/137880043 for i := range s { s[i] = 0 + // or + // Seebs suggested we might see even more crashes :) + // but since it will be slow (no memclr), we'll leave the default 0 for now. + //s[i] = -2 } } // keep this around if we need to activate out-of-mmap memory access again. @@ -632,21 +686,57 @@ func badgerKey(index, field, view string, shard uint64, roaringContainerKey uint prefix := badgerPrefix(index, field, view, shard) ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey)) - return append(prefix, ckey...) + bkey := append(prefix, ckey...) + MustValidateKey(bkey) + return bkey +} + +var ckeyPartExpected = []byte(";ckey@") + +// MustValidatekey will panic on a bad badgerKey with an informative message. +func MustValidateKey(bkey []byte) { + n := len(bkey) + if n < 56 { + panic(fmt.Sprintf("bkey too short min size is 56 but we see %v in '%v'", n, string(bkey))) + } + beforeCkey := bkey[n-26 : n-20] + if !bytes.Equal(beforeCkey, ckeyPartExpected) { + panic(fmt.Sprintf(`bkey did not have expected ";ckey@" at 26 bytes from the end of the bkey '%v'; instead had '%v'`, string(bkey), string(beforeCkey))) + } +} + +func shardFromBadgerKey(bkey []byte) (shard uint64) { + MustValidateKey(bkey) + + n := len(bkey) + // idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615 -> idx:'i';fld:'f';vw:'standard';shd:'1 + by := bkey[:n-27] + beg := bytes.LastIndex(by, []byte("'")) + if beg == -1 { + panic(fmt.Sprintf("bad bkey='%v' did not have single quote to being shard decoding", string(bkey))) + } + parseMe := string(by[beg+1:]) + shard, err := strconv.ParseUint(parseMe, 10, 64) + if err != nil { + panic(fmt.Sprintf("could not parse parseMe '%v' in strconv.ParseUint(), error: '%v'", parseMe, err)) + } + return shard } // badgerKeyAndPrefix returns the equivalent of badgerKey() and badgerPrefix() calls. func badgerKeyAndPrefix(index, field, view string, shard uint64, roaringContainerKey uint64) (key, prefix []byte) { prefix = badgerPrefix(index, field, view, shard) ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey)) - return append(prefix, ckey...), prefix + bkey := append(prefix, ckey...) + MustValidateKey(bkey) + return bkey, prefix } var _ = badgerKeyAndPrefix // keep linter happy // badgerKeyExtractContainerKey extracts the containerKey from bkey. func badgerKeyExtractContainerKey(bkey []byte) (containerKey uint64) { - + MustValidateKey(bkey) // The zero padding means that the container-key is always the last 20 bytes of the bkey. // // Be sure to catch the problematic case of a user passing in only a prefix. A prefix @@ -665,11 +755,15 @@ func badgerKeyExtractContainerKey(bkey []byte) (containerKey uint64) { return } +func badgerAllShardPrefix(index, field, view string) []byte { + return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:", index, field, view)) +} + // badgerPrefix returns everything from badgerKey up to and // including the '@' fune in a badger key. The prefix excludes the roaring container key itself. // NB must be kept in sync with badgerKey() and badgerKeyExtractContainerKey(). func badgerPrefix(index, field, view string, shard uint64) []byte { - return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%x';ckey@", index, field, view, shard)) + return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@", index, field, view, shard)) } // badgerIndexOnlyPrefix returns a prefix suitable for DeleteIndex and a key-scan to @@ -683,6 +777,11 @@ func badgerIndexOnlyPrefix(indexName string) []byte { return []byte(fmt.Sprintf("idx:'%v';", indexName)) } +// same for deleting a whole field. +func badgerFieldPrefix(index, field string) []byte { + return []byte(fmt.Sprintf("idx:'%v';fld:'%v';", index, field)) +} + // Container returns the requested roaring.Container, selected by fragment and ckey func (tx *BadgerTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) { @@ -869,6 +968,38 @@ func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64 return exists, err } +func (tx *BadgerTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { + + prefix := badgerAllShardPrefix(index, field, view) + + bi := NewBadgerIterator(tx, prefix) + defer bi.Close() + bi.Seek(prefix) + if !bi.it.Valid() { + return + } + lastShard := uint64(0) + firstDone := false + for bi.Next() { + item := bi.it.Item() + key := item.Key() + shard := shardFromBadgerKey(key) + if firstDone { + if shard != lastShard { + sliceOfShards = append(sliceOfShards, shard) + } + lastShard = shard + } else { + // first time + lastShard = shard + firstDone = true + sliceOfShards = append(sliceOfShards, shard) + } + + } + return +} + // key is the container key for the first roaring Container // roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will // return the first container at or after key. found will be true if a @@ -877,10 +1008,10 @@ func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64 // BadgerTx notes: We auto-stop at the end of this shard, not going beyond. func (tx *BadgerTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { - // needle example: "index:'i';field:'f';view:'v';shard:'0';key@00000000000000000000" + // needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000" needle := badgerKey(index, field, view, shard, firstRoaringContainerKey) - // prefix example: "index:'i';field:'f';view:'v';shard:'0';key@" + // prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@" prefix := badgerPrefix(index, field, view, shard) bi := NewBadgerIterator(tx, prefix) @@ -892,7 +1023,10 @@ func (tx *BadgerTx) ContainerIterator(index, field, view string, shard uint64, f if !bi.it.ValidForPrefix(prefix) { return bi, false, nil } - return bi, true, nil + item := bi.it.Item() + // have to compare b/c badger might give us valid iterator + // that is past our needle if needle isn't present. + return bi, bytes.Equal(item.Key(), needle), nil } // BadgerIterator is the iterator returned from a BadgerTx.ContainerIterator() call. @@ -917,12 +1051,6 @@ func NewBadgerIterator(tx *BadgerTx, prefix []byte) (bi *BadgerIterator) { tx.Db.muOpenTxIt.Lock() defer tx.Db.muOpenTxIt.Unlock() - defer func() { - r := recover() - if r != nil { - panic(r) - } - }() opts := badger.DefaultIteratorOptions opts.PrefetchValues = false // else by default, pre-fetches the 1st 100 values, which would be slow. opts.Reverse = false @@ -1125,6 +1253,7 @@ func (tx *BadgerTx) Count(index, field, view string, shard uint64) (uint64, erro } // Max is the maximum bit-value in your bitmap. +// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does. func (tx *BadgerTx) Max(index, field, view string, shard uint64) (uint64, error) { prefix := badgerPrefix(index, field, view, shard) @@ -1133,7 +1262,10 @@ func (tx *BadgerTx) Max(index, field, view string, shard uint64) (uint64, error) it := NewBadgerReverseIterator(tx, prefix, seekto) // this iterator is still open, when we commit/discard tx. defer it.Close() - hb, rc := it.Value() + if !it.it.Valid() { + return 0, nil + } + hb, rc := it.Value() // getting it returns invalid, as in empty iterator lb := rc.Max() return hb<<16 | uint64(lb), nil @@ -1183,18 +1315,22 @@ func (tx *BadgerTx) UnionInPlace(index, field, view string, shard uint64, others } // CountRange returns the count of hot bits in the start, end range on the fragment. +// roaring.countRange counts the number of bits set between [start, end). func (tx *BadgerTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { + if start >= end { + return 0, nil + } + skey := highbits(start) ekey := highbits(end) citer, found, err := tx.ContainerIterator(index, field, view, shard, skey) + _ = found panicOn(err) - defer citer.Close() // doesn't seem to be getting called. - if !found { - return 0, nil - } + defer citer.Close() + // If range is entirely in one container then just count that range. if skey == ekey { citer.Next() @@ -1284,6 +1420,7 @@ func (tx *BadgerTx) OffsetRange(index, field, view string, shard, offset, start, bkey := item.Key() k := badgerKeyExtractContainerKey(bkey) + // >= hi1 is correct b/c endx cannot have any lowbits set. if uint64(k) >= hi1 { break } @@ -1309,7 +1446,7 @@ func (tx *BadgerTx) IncrementOpN(index, field, view string, shard uint64, change // ImportRoaringBits handles deletes by setting clear=true. // rowSet[rowID] returns the number of bit changed on that rowID. -func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { +func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { n := itr.Len() if n == 0 { return @@ -1367,7 +1504,7 @@ func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, i changed += changes rowSet[currRow] -= changes - if newC.N() == 0 { + if tx.DeleteEmptyContainer && newC.N() == 0 { err = tx.RemoveContainer(index, field, view, shard, itrKey) if err != nil { return @@ -1399,7 +1536,7 @@ func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, i continue } - newC := oldC.UnionInPlace(synthC) + newC := roaring.Union(oldC, synthC) // UnionInPlace was giving us crashes on overly large containers. if roaring.ContainerType(newC) == containerBitmap { newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it. @@ -1447,6 +1584,10 @@ const ( func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) { + if len(v) == 0 { + return nil + } + // For safety we copy v, since it lives in BadgerDB's memory-mapped vlog-file, // and Badger will recycle it after tx ends with rollback or commit. // We copy into Go runtime GC managed memory. Technically we don't need @@ -1456,11 +1597,19 @@ func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) { // TODO: performance tuning might want w := v here, if we can guarantee no access to memory past the Tx lifetime. // // Problem is, at least some tests appear to not respect transaction boundaries... + // + // Seebs suggested this nice variation: we could use individual mmaps for these + // copies, which would be unusable in production, but workable for testing, and then unmap them, + // which would get us probable segfaults on future accesses to them. + // w := make([]byte, len(v)) - copy(w, v) // green go test -v -run TestAPI_ImportColumnAttrs + copy(w, v) + // the copy above makes green: // green go test -v -run TestAPI_ImportColumnAttrs //w := v // if instead of append we use v directly, it causes red: go test -v -run TestAPI_ImportColumnAttrs // register w so we can catch out-of-tx memory access + tx.acMu.Lock() + defer tx.acMu.Unlock() tx.ourAllocs = append(tx.ourAllocs, w) switch typ { @@ -1483,35 +1632,39 @@ func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) { // fromArray16 converts to an 8KB page func fromArray16(a []uint16) []byte { + if len(a) == 0 { + return []byte{} + } + if len(a) > 4096 { + panic(fmt.Sprintf("cannot put more than 4096 integers into an array container: %v too big", len(a))) + } return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2] } // fromArray64 converts to an 8KB page func fromArray64(a []uint64) []byte { + if len(a) == 0 { + return []byte{} + } return (*[8192]byte)(unsafe.Pointer(&a[0]))[:8192:8192] } // fromInterval16 converts to 8KB page func fromInterval16(a []roaring.Interval16) []byte { + if len(a) == 0 { + return []byte{} + } + if len(a) > 2048 { + panic(fmt.Sprintf("cannot put more than 2048 roaring.Interval16 into a container: %v too big", len(a))) + } return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4] } -// badgerKey method on fragment creates a query key in the -// standard format by invoking the top level badgerKey with -// the container key being highbits(rowID * ShardWidth). -// -// Commented out for now only to keep the golangci-lint happy, -// as it has no users at the moment. -//func (f *fragment) badgerKey(rowID uint64) []byte { -// hi0 := highbits(rowID * ShardWidth) -// return badgerKey(f.index, f.field, f.view, f.shard, hi0) -//} - // StringifiedBadgerKeys returns a string with all the container // keys available in badger. func (w *BadgerDBWrapper) StringifiedBadgerKeys(optionalUseThisTx Tx) (r string) { if optionalUseThisTx == nil { - tx := w.NewBadgerTx(!writable) + tx := w.NewBadgerTx(!writable, "") defer tx.Rollback() r = stringifiedBadgerKeysTx(tx) return @@ -1548,6 +1701,10 @@ func (tx *BadgerTx) countBitsSet(bkey []byte) (n int) { return } +func (tx *BadgerTx) Dump() { + fmt.Printf("%v\n", stringifiedBadgerKeysTx(tx)) +} + // stringifiedBadgerKeysTx reports all the badger keys and a // corresponding blake3 hash viewable by txn within the entire // badger database. @@ -1619,6 +1776,23 @@ func asInts(a []uint64) (r []int) { return } +var _ = zeroKeyContainerAsString // happy linter + +// for debugging +func zeroKeyContainerAsString(ct *roaring.Container) (r string) { + cts := roaring.NewSliceContainers() + cts.Put(0, ct) + rbm := &roaring.Bitmap{Containers: cts} + r = fmt.Sprintf("[%v]:", containerTypeNames[roaring.ContainerType(ct)]) + bitmapAsString(rbm) + return +} + +var containerTypeNames = map[byte]string{ + containerArray: "array", + containerBitmap: "bitmap", + containerRun: "run", +} + func bitmapAsString(rbm *roaring.Bitmap) (r string) { r = "c(" slc := rbm.Slice() @@ -1699,3 +1873,107 @@ func dirAsString(path string) (r string) { } var _ = dirAsString // happy linter + +func (w *BadgerDBWrapper) DeleteField(index, field, fieldPath string) error { + + // under blue-green roaring_badger, the directory will not be found, b/c roaring will have + // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: + // "If the path does not exist, RemoveAll returns nil (no error)" + err := os.RemoveAll(fieldPath) + if err != nil { + return errors.Wrap(err, "removing directory") + } + prefix := badgerFieldPrefix(index, field) + return w.DeletePrefix(prefix) +} + +func (w *BadgerDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { + prefix := badgerPrefix(index, field, view, shard) + return w.DeletePrefix(prefix) +} + +func (w *BadgerDBWrapper) DeletePrefix(prefix []byte) error { + w.muDb.Lock() + defer w.muDb.Unlock() + + // a) do key-ony iteration, no value fetch; + // + // b) do deletes in large batches, to avoid alot of txn overhead; + // per recommendation https://github.com/dgraph-io/badger/issues/598 + // + // c) we do not, at present, try to maintain one large + // transaction with all the keys in a index in it. Because + // there can be too many keys. Hence the index will disappear + // in chucks of 100K keys, not atomically-all-at-once. + + noMoreKeysWithPrefix := false + const maxDeletesPerTxn = 100000 + + for !noMoreKeysWithPrefix { + err := w.db.Update(func(txn *badger.Txn) error { + o := badger.DefaultIteratorOptions + o.AllVersions = false + o.PrefetchValues = false // key-only iteration, no values. + + // note: panic: Unclosed iterator at time of Txn.Discard ? panic on segfault here? + // This means we messed up and Closed() the Database already; too early. + it := txn.NewIterator(o) + + defer it.Close() + n := 0 + goners := make([][]byte, 0, maxDeletesPerTxn) + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + + // KeyCopy() is required; Key() means corruption and possible segfault. + key := it.Item().KeyCopy(nil) + goners = append(goners, key) + n++ + if n >= maxDeletesPerTxn { + break + } + } + if !it.ValidForPrefix(prefix) { + noMoreKeysWithPrefix = true // done with the full delete of up to maxDeletesPerTxn + } + for _, key := range goners { + if err := txn.Delete(key); err != nil { + return err + } + } + return nil // auto-commit happens + }) + // err back from Update can be ErrConflict in case of + // a conflict. Badger docs: "Depending on the state + // of your application, you have the option to + // retry the operation if you receive this error." + panicOn(err) + + } // end for: proceed to next bath of 100K keys + + // Finally, run a garbage collection to delete values from the value log. + // + // "Only one GC is allowed at a time. If another value log GC + // is running, or DB has been closed, this would return an ErrRejected." + // -- https://godoc.org/github.com/dgraph-io/badger#DB.RunValueLogGC + // Still, we don't see a mutex inside the RunValueLogGC code, so + // lock muGC just to be sure. + w.muGC.Lock() + defer w.muGC.Unlock() + _ = w.db.RunValueLogGC(0.5) + + return nil +} + +func (tx *BadgerTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { + + rbm, err := tx.RoaringBitmap(index, field, view, shard) + if err != nil { + return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") + } + var buf bytes.Buffer + sz, err = rbm.WriteTo(&buf) + if err != nil { + return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") + } + return ioutil.NopCloser(&buf), sz, err +} diff --git a/badger_test.go b/badger_test.go index 4f2f09f52..821356867 100644 --- a/badger_test.go +++ b/badger_test.go @@ -12,12 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +// explanation of build tags: +// // badgerdb builds but won't run in 32-bit 386 world, as of 2020 July 20. // See https://github.com/dgraph-io/badger/issues/1384 for any progress. // What we see is that the value-log allocations immediately run out of // memory. So we turn off 386 with a build tag to keep the .circleci happy. +// +// gendebug_test will have a TestMain if build tag generationdebug is on, +// so we avoid conflicting with that debug scenario. // +build !386 +// +build !generationdebug package pilosa @@ -40,7 +46,7 @@ var _ = &roaring.Bitmap{} func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) { - tx := dbwrap.NewBadgerTx(!writable) + tx := dbwrap.NewBadgerTx(!writable, index) defer tx.Rollback() exists, err := tx.Contains(index, field, view, shard, bitvalue) panicOn(err) @@ -53,7 +59,7 @@ func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) { - tx := dbwrap.NewBadgerTx(!writable) + tx := dbwrap.NewBadgerTx(!writable, index) defer tx.Rollback() exists, err := tx.Contains(index, field, view, shard, bitvalue) panicOn(err) @@ -64,7 +70,7 @@ func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view str } func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) { - tx := dbwrap.NewBadgerTx(writable) + tx := dbwrap.NewBadgerTx(writable, index) // add a bit changed, err := tx.Add(index, field, view, shard, doBatched, putme) @@ -82,14 +88,14 @@ func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, } func badgerDBMustDeleteBitvalueContainer(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) { - tx := dbwrap.NewBadgerTx(writable) + tx := dbwrap.NewBadgerTx(writable, index) hi := highbits(putme) panicOn(tx.RemoveContainer(index, field, view, shard, hi)) panicOn(tx.Commit()) } func badgerDBMustDeleteBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) { - tx := dbwrap.NewBadgerTx(writable) + tx := dbwrap.NewBadgerTx(writable, index) _, err := tx.Remove(index, field, view, shard, putme) panicOn(err) panicOn(tx.Commit()) @@ -99,7 +105,7 @@ func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func() var err error fn := badgerPath(path) panicOn(os.RemoveAll(fn)) - w, err = newBadgerDBWrapper(path) + w, err = globalBadgerReg.newBadgerDBWrapper(path) panicOn(err) // verify it is empty @@ -109,6 +115,7 @@ func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func() } return w, func() { + w.Close() // stop any started background GC goroutine. os.RemoveAll(fn) } } @@ -118,8 +125,8 @@ func TestBadger_SetBitmap(t *testing.T) { dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SetBitmap") defer clean() defer dbwrap.Close() - tx := dbwrap.NewBadgerTx(writable) index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index) bitvalue := uint64(0) changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue) if changed <= 0 { @@ -140,7 +147,7 @@ func TestBadger_SetBitmap(t *testing.T) { // commited, so should be visible outside the txn // - tx2 := dbwrap.NewBadgerTx(!writable) + tx2 := dbwrap.NewBadgerTx(!writable, index) exists, err = tx2.Contains(index, field, view, shard, bitvalue) panicOn(err) if !exists { @@ -159,9 +166,9 @@ func TestBadger_OffsetRange(t *testing.T) { dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SetBitmap") defer clean() defer dbwrap.Close() - tx := dbwrap.NewBadgerTx(writable) - index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index) + bitvalue := uint64(1 << 20) changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue) if changed <= 0 { @@ -194,7 +201,7 @@ func TestBadger_OffsetRange(t *testing.T) { start := uint64(0 << 16) endx := bitvalue + 1<<16 - tx2 := dbwrap.NewBadgerTx(!writable) + tx2 := dbwrap.NewBadgerTx(!writable, index) rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx) panicOn(err) tx2.Rollback() @@ -208,7 +215,7 @@ func TestBadger_OffsetRange(t *testing.T) { // now offset by 2M offset = uint64(2 << 20) - tx3 := dbwrap.NewBadgerTx(!writable) + tx3 := dbwrap.NewBadgerTx(!writable, index) rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx) panicOn(err) tx3.Rollback() @@ -236,7 +243,7 @@ func TestBadger_Count_on_many_containers(t *testing.T) { badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } - tx := dbwrap.NewBadgerTx(writable) + tx := dbwrap.NewBadgerTx(writable, index) defer tx.Rollback() n, err := tx.Count(index, field, view, shard) @@ -252,7 +259,7 @@ func TestBadger_Count_dense_containers(t *testing.T) { defer dbwrap.Close() index, field, view, shard := "i", "f", "v", uint64(0) - tx := dbwrap.NewBadgerTx(writable) + tx := dbwrap.NewBadgerTx(writable, index) expected := 0 // can't do more than about 100k writes per badger txn by default, so @@ -280,9 +287,9 @@ func TestBadger_ContainerIterator_on_empty(t *testing.T) { dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator") defer clean() defer dbwrap.Close() - tx := dbwrap.NewBadgerTx(!writable) - defer tx.Rollback() index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(!writable, index) + defer tx.Rollback() bitvalue := uint64(0) citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue) panicOn(err) @@ -298,9 +305,9 @@ func TestBadger_ContainerIterator_on_one_bit(t *testing.T) { dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit") defer clean() defer dbwrap.Close() - tx := dbwrap.NewBadgerTx(writable) - defer tx.Rollback() index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index) + defer tx.Rollback() bitvalue := uint64(42) @@ -398,9 +405,9 @@ func TestBadger_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) { dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit") defer clean() defer dbwrap.Close() - tx := dbwrap.NewBadgerTx(writable) - defer tx.Rollback() index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index) + defer tx.Rollback() putme := uint64(1<<16) + 3 // in the key:1 container searchme := putme + 1 @@ -453,9 +460,9 @@ func TestBadger_ContainerIterator_empty_iteration_loop(t *testing.T) { dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_empty_iteration_loop") defer clean() defer dbwrap.Close() - tx := dbwrap.NewBadgerTx(writable) - defer tx.Rollback() index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index) + defer tx.Rollback() putme := uint64(1<<16) + 3 // in the key:1 container searchme := uint64(1 << 17) // in the next container, key:2 @@ -503,9 +510,9 @@ func TestBadger_ForEach_on_one_bit(t *testing.T) { dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit") defer clean() defer dbwrap.Close() - tx := dbwrap.NewBadgerTx(writable) - defer tx.Rollback() index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index) + defer tx.Rollback() bitvalue := uint64(42) @@ -563,7 +570,7 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) { badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) // delete, but rollback instead of commit - tx := dbwrap.NewBadgerTx(writable) + tx := dbwrap.NewBadgerTx(writable, index) hi := highbits(putme) panicOn(tx.RemoveContainer(index, field, view, shard, hi)) tx.Rollback() @@ -572,7 +579,7 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) { badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) // c) within one Tx, after delete it should be gone as viewed within the txn. - tx = dbwrap.NewBadgerTx(writable) + tx = dbwrap.NewBadgerTx(writable, index) hi = highbits(putme) exists, err := tx.Contains(index, field, view, shard, putme) @@ -624,7 +631,7 @@ func TestBadger_Remove_one_bit_test(t *testing.T) { badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) // delete, but rollback instead of commit - tx := dbwrap.NewBadgerTx(writable) + tx := dbwrap.NewBadgerTx(writable, index) hi, lo := highbits(putme), lowbits(putme) _, _ = hi, lo _, err := tx.Remove(index, field, view, shard, hi) @@ -635,7 +642,7 @@ func TestBadger_Remove_one_bit_test(t *testing.T) { badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) // c) within one Tx, after delete it should be gone as viewed within the txn. - tx = dbwrap.NewBadgerTx(writable) + tx = dbwrap.NewBadgerTx(writable, index) exists, err := tx.Contains(index, field, view, shard, putme) panicOn(err) @@ -724,7 +731,7 @@ func TestBadger_Max_on_many_containers(t *testing.T) { badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } - tx := dbwrap.NewBadgerTx(!writable) + tx := dbwrap.NewBadgerTx(!writable, index) defer tx.Rollback() max, err := tx.Max(index, field, view, shard) @@ -742,7 +749,7 @@ func TestBadger_Min_on_many_containers(t *testing.T) { index, field, view, shard := "i", "f", "v", uint64(0) // verify no containers flag works - tx := dbwrap.NewBadgerTx(!writable) + tx := dbwrap.NewBadgerTx(!writable, index) min, containersExist, err := tx.Min(index, field, view, shard) _ = min panicOn(err) @@ -759,7 +766,7 @@ func TestBadger_Min_on_many_containers(t *testing.T) { badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } - tx = dbwrap.NewBadgerTx(!writable) + tx = dbwrap.NewBadgerTx(!writable, index) defer tx.Rollback() min, containersExist, err = tx.Min(index, field, view, shard) @@ -780,7 +787,7 @@ func TestBadger_CountRange_on_many_containers(t *testing.T) { index, field, view, shard := "i", "f", "v", uint64(0) // verify no containers flag works - tx := dbwrap.NewBadgerTx(!writable) + tx := dbwrap.NewBadgerTx(!writable, index) n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64) panicOn(err) if n != 0 { @@ -796,7 +803,7 @@ func TestBadger_CountRange_on_many_containers(t *testing.T) { badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } - tx = dbwrap.NewBadgerTx(!writable) + tx = dbwrap.NewBadgerTx(!writable, index) defer tx.Rollback() n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64) @@ -824,7 +831,7 @@ func TestBadger_CountRange_middle_container(t *testing.T) { badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } - tx := dbwrap.NewBadgerTx(!writable) + tx := dbwrap.NewBadgerTx(!writable, index) defer tx.Rollback() // pick out just the middle container with the 1 bit set on it. @@ -849,7 +856,7 @@ func TestBadger_CountRange_many_middle_container(t *testing.T) { badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } - tx := dbwrap.NewBadgerTx(!writable) + tx := dbwrap.NewBadgerTx(!writable, index) defer tx.Rollback() // get them all @@ -880,7 +887,7 @@ func TestBadger_UnionInPlace(t *testing.T) { badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) } - tx2 := dbwrap.NewBadgerTx(!writable) + tx2 := dbwrap.NewBadgerTx(!writable, index) n, err := tx2.Count(index, field, view, shard) panicOn(err) if n != 2 { @@ -895,7 +902,7 @@ func TestBadger_UnionInPlace(t *testing.T) { } mustAddR(others3.Add(4 << 16)) // outside the 2<<16 container - tx := dbwrap.NewBadgerTx(writable) + tx := dbwrap.NewBadgerTx(writable, index) defer tx.Rollback() err = tx.UnionInPlace(index, field, view, shard, others, others2, others3) panicOn(err) @@ -920,7 +927,7 @@ func TestBadger_RoaringBitmap(t *testing.T) { putme := expected badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - tx := dbwrap.NewBadgerTx(!writable) + tx := dbwrap.NewBadgerTx(!writable, index) defer tx.Rollback() rbm, err := tx.RoaringBitmap(index, field, view, shard) @@ -951,10 +958,8 @@ func TestBadger_reverse_badger_iterator_and_prefix_valid(t *testing.T) { return nil }) panicOn(err) - //vv("stringifiedBadgerKeys(db) = '%v'", stringifiedBadgerKeys(dbwrap.db)) - // allkeys:["a:0", "a:1", "a:2", "b:0", "b:1", "b:2", "c:0", "c:1", "c:2", ]' - tx := dbwrap.NewBadgerTx(!writable) + tx := dbwrap.NewBadgerTx(!writable, "no-index-avail") prefix := []byte("b:") it := NewBadgerIterator(tx, prefix) @@ -1014,10 +1019,8 @@ func TestBadger_just_reverse_badger_iterator_and_prefix_valid(t *testing.T) { return nil }) panicOn(err) - //vv("stringifiedBadgerKeys(db) = '%v'", stringifiedBadgerKeys(dbwrap.db)) - // allkeys:["a:0", "a:1", "a:2", "b:0", "b:1", "b:2", "c:0", "c:1", "c:2", ]' - tx := dbwrap.NewBadgerTx(!writable) + tx := dbwrap.NewBadgerTx(!writable, "no-index-avail") seekto := []byte("c:") prefix := []byte("b:") @@ -1046,9 +1049,10 @@ func TestBadger_ImportRoaringBits(t *testing.T) { dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits") defer clean() defer dbwrap.Close() - tx := dbwrap.NewBadgerTx(writable) - defer tx.Rollback() index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index) + defer tx.Rollback() + tx.DeleteEmptyContainer = true // traditional badger Tx behavior, but not Roaring. //bitvalue := uint64(42) @@ -1062,7 +1066,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) { clear := false logme := false - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize) + changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) _ = rowSet if changed != len(bits) { panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) @@ -1079,7 +1083,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) { // now test the union in place with the same set gives no change. - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize) + changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) _ = rowSet if changed != 0 { panic(fmt.Sprintf("should have not changed any bits on the second import, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) @@ -1103,7 +1107,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) { itr, err := roaring.NewRoaringIterator(data) panicOn(err) - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize) + changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) _ = rowSet if changed != 1 { panic(fmt.Sprintf("should have changed 1 bit: '%v', rowSet='%#v', err='%v'", changed, rowSet, err)) @@ -1128,9 +1132,9 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits_set_nonoverlapping_bits") defer clean() defer dbwrap.Close() - tx := dbwrap.NewBadgerTx(writable) - defer tx.Rollback() index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index) + defer tx.Rollback() // get some roaring bits, get an itr RoaringIterator from them rowSize := uint64(0) @@ -1148,7 +1152,7 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { clear := false logme := false - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize) + changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) _ = rowSet if changed != len(bits) { panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) @@ -1165,7 +1169,7 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { // now import the 2nd, overlapping set and set them. - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize) + changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil) _ = rowSet if changed != 4 { panic(fmt.Sprintf("should have changed 2 bits: the 1 and the 3, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) @@ -1178,9 +1182,9 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits_clear_nonoverlapping_bits") defer clean() defer dbwrap.Close() - tx := dbwrap.NewBadgerTx(writable) - defer tx.Rollback() index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index) + defer tx.Rollback() // get some roaring bits, get an itr RoaringIterator from them rowSize := uint64(0) @@ -1198,7 +1202,7 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { clear := false logme := false - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize) + changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) _ = rowSet if changed != len(bits) { panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) @@ -1216,7 +1220,7 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { // now import the 2nd overlapping set and clear them. clear = true - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize) + changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil) _ = rowSet if changed != 2 { panic(fmt.Sprintf("should have changed 1 bit: the 2, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) @@ -1252,9 +1256,9 @@ func TestBadger_DeleteIndex(t *testing.T) { dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex") defer clean() defer dbwrap.Close() - tx := dbwrap.NewBadgerTx(writable) - bitvalue := uint64(777) index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index) + bitvalue := uint64(777) bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} for _, v := range bits { changed, err := tx.Add(index, field, view, shard, doBatched, v) @@ -1290,7 +1294,7 @@ func TestBadger_DeleteIndex(t *testing.T) { err = dbwrap.DeleteIndex(index) panicOn(err) - tx = dbwrap.NewBadgerTx(!writable) + tx = dbwrap.NewBadgerTx(!writable, index2) defer tx.Rollback() exists, err = tx.Contains(index2, field, view, shard, bitvalue) panicOn(err) @@ -1314,9 +1318,9 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) { dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex_over100k") defer clean() defer dbwrap.Close() - tx := dbwrap.NewBadgerTx(writable) - bitvalue := uint64(777) index, field, view, shard := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index) + bitvalue := uint64(777) limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction. //limit := uint64(101) for v := uint64(1); v < limit; v++ { @@ -1328,7 +1332,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) { panicOn(err) if v%100000 == 0 { panicOn(tx.Commit()) - tx = dbwrap.NewBadgerTx(writable) + tx = dbwrap.NewBadgerTx(writable, index) } } @@ -1345,7 +1349,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) { err = dbwrap.DeleteIndex(index) panicOn(err) - tx = dbwrap.NewBadgerTx(!writable) + tx = dbwrap.NewBadgerTx(!writable, index2) defer tx.Rollback() exists, err := tx.Contains(index2, field, view, shard, bitvalue) panicOn(err) @@ -1445,3 +1449,134 @@ func mustAddR(changed bool, err error) { func mustRemove(changeCount int, err error) { panicOn(err) } + +func TestBadger_DeleteFragment(t *testing.T) { + + // setup + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteFragment") + defer clean() + defer dbwrap.Close() + index, field, view, shard0 := "i", "f", "v", uint64(0) + tx := dbwrap.NewBadgerTx(writable, index) + + shard1 := uint64(1) + + bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} + shards := []uint64{shard0, shard1} + for _, s := range shards { + for _, v := range bits { + changed, err := tx.Add(index, field, view, s, doBatched, v) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + } + } + + for _, s := range shards { + for _, v := range bits { + exists, err := tx.Contains(index, field, view, s, v) + panicOn(err) + if !exists { + panic("ARG bitvalue was NOT SET!!!") + } + } + } + err := tx.Commit() + panicOn(err) + + // end of setup + + survivor := shard0 + victim := shard1 + err = dbwrap.DeleteFragment(index, field, view, victim, nil) + panicOn(err) + + tx = dbwrap.NewBadgerTx(!writable, index) + defer tx.Rollback() + + for _, s := range shards { + for _, v := range bits { + exists, err := tx.Contains(index, field, view, s, v) + panicOn(err) + if s == survivor { + if !exists { + panic(fmt.Sprintf("ARG survivor died : bit %v", v)) + } + } else if s == victim { // victim, should have been deleted + if exists { + panic(fmt.Sprintf("ARG victim lived : bit %v", v)) + } + } + } + } +} + +func TestBadger_shardFromBadgerKey(t *testing.T) { + if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615")) != 1 { + panic("problem") + } + if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'0';ckey@18446744073709551615")) != 0 { + panic("problem") + } + if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@18446744073709551615")) != 18446744073709551615 { + panic("problem") + } + + func() { + defer func() { + r := recover() + if r == nil { + panic("should have panic-ed") + } + }() + // called for the panic of a short ckey, only 19 bytes instead of 20 + shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@1844674407370955161")) + }() + +} + +func TestBadger_SliceOfShards(t *testing.T) { + + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SliceOfShards") + defer clean() + defer dbwrap.Close() + index, field, view := "i", "f", "v" + shards := []uint64{0, 1, 2, 3, 1000001, 2000001} + putme := uint64(179) + for _, shard := range shards { + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + } + tx := dbwrap.NewBadgerTx(!writable, index) + defer tx.Rollback() + + slc, err := tx.SliceOfShards(index, field, view, "") + panicOn(err) + for i := range shards { + if shards[i] != slc[i] { + panic(fmt.Sprintf("expected at i=%v that slc[i]=%v = shards[i]=%v", i, slc[i], shards[i])) + } + } +} + +func reportTestBadgersNeedingClose() { + globalBadgerReg.mu.Lock() + defer globalBadgerReg.mu.Unlock() + n := len(globalBadgerReg.mp) + if n > 0 { + AlwaysPrintf("*** these badgers are still open (n=%v):", n) + i := 0 + for w := range globalBadgerReg.mp { + AlwaysPrintf("i=%v, w p=%p stack:\n%v\n\n", i, w, w.startStack) + i++ + } + } +} + +var _ = reportTestBadgersNeedingClose // happy linter + +func TestMain(m *testing.M) { + ret := m.Run() + //reportTestBadgersNeedingClose() + os.Exit(ret) +} diff --git a/bluegreentx.go b/bluegreentx.go index dd58c69fd..3084a2390 100644 --- a/bluegreentx.go +++ b/bluegreentx.go @@ -15,28 +15,65 @@ package pilosa import ( + "bytes" "fmt" + "io" + "reflect" + "sort" + "sync" "github.com/pilosa/pilosa/v2/roaring" ) // blueGreenTx runs two Tx together and notices differences in their output. // By convention, the 'b' Tx is the output that is returned to caller. +// +// Warning: DATA RACES are expected if RoaringTx is one side of the Tx pair. +// The checkDatabase() call will do reads of the fragments at Commit/Rollback, +// while the snapshotqueue may be doing writes. +// +// Do not run with go test -race and expect it to be race free. +// type blueGreenTx struct { a Tx b Tx // b's output is returned + as string + bs string + idx *Index + + checker blueGreenChecker + mu sync.Mutex + rollbackOrCommitDone bool } func newBlueGreenTx(a, b Tx, idx *Index) *blueGreenTx { - return &blueGreenTx{a: a, b: b, idx: idx} + as := a.Type() + bs := b.Type() + return &blueGreenTx{a: a, b: b, idx: idx, as: as, bs: bs} } var _ = newBlueGreenTx // keep linter happy var _ Tx = (*blueGreenTx)(nil) +func (c *blueGreenTx) Type() string { + return c.a.Type() + "_" + c.b.Type() +} + +var blueGreenTxDumpMut sync.Mutex + +func (c *blueGreenTx) Dump() { + blueGreenTxDumpMut.Lock() + defer blueGreenTxDumpMut.Unlock() + fmt.Printf("%v blueGreenTx.Dump ============== \n", FileLine(2)) + fmt.Printf("A(%v) Dump:\n", c.as) + c.a.Dump() + fmt.Printf("B(%v) Dump:\n", c.bs) + c.b.Dump() +} + func (c *blueGreenTx) Readonly() bool { a := c.a.Readonly() b := c.b.Readonly() @@ -47,6 +84,8 @@ func (c *blueGreenTx) Readonly() bool { } func (c *blueGreenTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { + c.checker.see(index, field, view, shard) + // TODO(jea): does this need to be different, to handle c.a iteration at the same time? return c.b.NewTxIterator(index, field, view, shard) } @@ -55,11 +94,94 @@ func (c *blueGreenTx) Pointer() string { } func (c *blueGreenTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { + c.checker.see(index, field, view, shard) c.a.IncrementOpN(index, field, view, shard, changedN) c.b.IncrementOpN(index, field, view, shard, changedN) } +// compareTxState is called for the first Commit or Rollback a blueGreenTx sees. +func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) { + here := fmt.Sprintf("%v/%v/%v/%v", index, field, view, shard) + aIter, aFound, aErr := c.a.ContainerIterator(index, field, view, shard, 0) + bIter, bFound, bErr := c.b.ContainerIterator(index, field, view, shard, 0) + if aErr == nil || aIter != nil { + defer aIter.Close() + } + if bErr == nil || bIter != nil { + defer bIter.Close() + } + + if aFound != bFound { + c.Dump() + panic(fmt.Sprintf("compareTxState[%v]: A(%v) ContainerIterator had aFound=%v, but B(%v) had bFound=%v; at '%v'", here, c.as, aFound, c.bs, bFound, stack())) + } + + if aErr != nil || bErr != nil { + if aErr != nil && bErr != nil { + c.Dump() + panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err '%v'; B(%v) reported err '%v' at %v", here, c.as, aErr, c.bs, bErr, stack())) + } + if aErr != nil { + c.Dump() + panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err %v at %v; but B(%v) did not", here, c.as, aErr, c.bs, stack())) + } + if bErr != nil { + c.Dump() + panic(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, stack())) + } + } + for aIter.Next() { + aKey, aValue := aIter.Value() + + if !bIter.Next() { + c.Dump() + panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, stack())) + } + bKey, bValue := bIter.Value() + if bKey != aKey { + AlwaysPrintf("problem in caller %v", Caller(2)) + c.Dump() + panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, stack())) // crashing here on TestBSIGroup_importValue + } + if err := aValue.BitwiseCompare(bValue); err != nil { + c.Dump() + panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack())) + } + } + // end checking everything in A, but does B have more? + if bIter.Next() { + bKey, _ := bIter.Value() + c.Dump() + panic(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, at %v", here, c.bs, bKey, c.as, stack())) + } +} + +func (c *blueGreenTx) checkDatabase() { + c.checker.mu.Lock() + defer c.checker.mu.Unlock() + + // seen() returns nil on 2nd or any further call, + // so only the first Commit() or Rollback() does this. + for index, fields := range c.checker.seen() { + for field, views := range fields { + for view, shards := range views { + for shard := range shards { + c.compareTxState(index, field, view, shard) + } + } + } + } +} + func (c *blueGreenTx) Rollback() { + c.mu.Lock() + defer c.mu.Unlock() + if c.rollbackOrCommitDone { + return + } + c.rollbackOrCommitDone = true + + c.checkDatabase() defer func() { if r := recover(); r != nil { AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack()) @@ -71,6 +193,13 @@ func (c *blueGreenTx) Rollback() { } func (c *blueGreenTx) Commit() error { + c.mu.Lock() + defer c.mu.Unlock() + if c.rollbackOrCommitDone { + return nil + } + c.rollbackOrCommitDone = true + c.checkDatabase() defer func() { if r := recover(); r != nil { AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack()) @@ -86,6 +215,7 @@ func (c *blueGreenTx) Commit() error { } func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack()) @@ -96,10 +226,18 @@ func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*r _, _ = a, errA b, errB := c.b.RoaringBitmap(index, field, view, shard) compareErrors(errA, errB) + + slcA := a.Slice() + slcB := b.Slice() + if !reflect.DeepEqual(slcA, slcB) { + panic("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!") + } + return b, errB } func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack()) @@ -116,6 +254,7 @@ func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uin } func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack()) @@ -126,17 +265,19 @@ func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key errB := c.b.PutContainer(index, field, view, shard, key, rc) compareErrors(errA, errB) - /* draft idea of how to check the full databases afterwards: - hashA := c.a.RootHashString() - hashB := c.b.RootHashString() - if hashA != hashB { - panic(fmt.Sprintf("hashA = '%v' but hashB = '%v'", hashA, hashB)) - } - */ return errB } -func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { +func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { + c.checker.see(index, field, view, shard) + + // these are the first port of call for debugging, so we leave them in. + // ================== begin save comments. + //c.checkDatabase() + //vv("got past database check at TOP of ImportRoaringBits") + //c.Dump() + //vv("done with top dump; clear=%v", clear) + // ================== end save comments. defer func() { if r := recover(); r != nil { AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack()) @@ -146,50 +287,38 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, // remember where the iterator started, so we can replay it a second time. rit2 := rit.Clone() + panicOn(err) - changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize) + changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) + changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize, data) - changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize) - - if changedA != changedB { - panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB)) - } - if len(rowSetA) != len(rowSetB) { - panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB)) - } - for k, va := range rowSetA { - vb, ok := rowSetB[k] - if !ok { - panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB)) + if len(data) == 0 { + // okay to check! otherwise we are in the fragment.fillFragmentFromArchive + // case where we know that RoaringTx.ImportRoaringBits changed and rowSet will + // be inaccurate. + if changedA != changedB { + panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB)) } - if va != vb { - panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb)) + if len(rowSetA) != len(rowSetB) { + panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB)) + } + for k, va := range rowSetA { + vb, ok := rowSetB[k] + if !ok { + panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB)) + } + if va != vb { + panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb)) + } } } - compareErrors(errA, errB) - - //compareDatabases(c.a, c.b) + c.checkDatabase() return changedB, rowSetB, errB } -/* // TODO: get a database-wide checksum working -func compareDatabases(a, b Tx) { - - index, field, view, shard := "i", "f", "v", uint64(0) - - ha, errA := a.WholeDatabaseBlake3Hash(index, field, view, shard) - panicOn(errA) - hb, errB := b.WholeDatabaseBlake3Hash(index, field, view, shard) - panicOn(errB) - - if ha != hb { - panic(fmt.Sprintf("a.WholeDatabaseBlake3Hash(%T) = '%v' but b.WholeDatabaseBlake3Hash(%T) = '%v'", a, ha, b, hb)) - } -} -*/ - func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack()) @@ -207,6 +336,7 @@ func (c *blueGreenTx) UseRowCache() bool { } func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see Add() panic '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, stack()) @@ -249,6 +379,7 @@ func compareErrors(errA, errB error) { } func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack()) @@ -263,6 +394,7 @@ func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint6 } func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack()) @@ -278,21 +410,81 @@ func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint } func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack()) panic(r) } }() - // TODO: need to return a blueGreenIterator too, that does close/next operations on both A and B. + ait, afound, errA := c.a.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) _, _, _ = ait, afound, errA + bit, bfound, errB := c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) compareErrors(errA, errB) - return bit, bfound, errB + // INVAR: errA == errB, so only need to check one. + if errB != nil { + // RoaringTx can return an iterator and an error, so be sure Close it we have it. + if ait != nil { + ait.Close() + } + if bit != nil { + bit.Close() + } + return nil, bfound, errB + } + // INVAR: errA == errB == nil + bgi := NewBlueGreenIterator(c, ait, bit) + return bgi, bfound, errB } +func NewBlueGreenIterator(tx *blueGreenTx, ait, bit roaring.ContainerIterator) *blueGreenIterator { + return &blueGreenIterator{ + tx: tx, + as: tx.as, + bs: tx.bs, + ait: ait, + bit: bit, + } +} + +type blueGreenIterator struct { + tx *blueGreenTx + as string + bs string + + ait roaring.ContainerIterator + bit roaring.ContainerIterator +} + +func (bgi *blueGreenIterator) Next() bool { + na := bgi.ait.Next() + nb := bgi.bit.Next() + if na != nb { + panic(fmt.Sprintf("na=%v(%v) != nb(%v)=%v", na, bgi.as, bgi.bs, nb)) + } + return nb +} + +func (bgi *blueGreenIterator) Value() (uint64, *roaring.Container) { + ka, ca := bgi.ait.Value() + kb, cb := bgi.bit.Value() + if ka != kb { + panic(fmt.Sprintf("ka=%v != kb=%v", ka, kb)) + } + err := ca.BitwiseCompare(cb) + panicOn(err) + return kb, cb +} +func (bgi *blueGreenIterator) Close() { + bgi.ait.Close() + bgi.bit.Close() +} + +// ForEach is read-only on the database, and so we only pass through to B. +// Avoids the side-effects of calling fn too many times. func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { defer func() { if r := recover(); r != nil { @@ -300,32 +492,30 @@ func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i panic(r) } }() - errA := c.a.ForEach(index, field, view, shard, fn) - _ = errA - errB := c.b.ForEach(index, field, view, shard, fn) - _ = errB + return c.b.ForEach(index, field, view, shard, fn) - compareErrors(errA, errB) - return errB } +// ForEachRange cannot change the database, and we also can't control +// the side effects of the fn() calls. So we only pass through to B, not A. +// No checker.see() is needed as well, because we are read-only. func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { + defer func() { if r := recover(); r != nil { AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack()) panic(r) } }() - errA := c.a.ForEachRange(index, field, view, shard, start, end, fn) - _ = errA - errB := c.b.ForEachRange(index, field, view, shard, start, end, fn) - _ = errB - compareErrors(errA, errB) - return errB + // calling fn will have side effects; can only call it the right number of times. + // so can't do this. + // errA := c.a.ForEachRange(index, field, view, shard, start, end, fn) + return c.b.ForEachRange(index, field, view, shard, start, end, fn) } func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, error) { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack()) @@ -342,6 +532,7 @@ func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, er } func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, error) { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack()) @@ -358,6 +549,7 @@ func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, erro } func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack()) @@ -374,6 +566,7 @@ func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool } func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack()) @@ -387,8 +580,11 @@ func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, othe } func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { + c.checker.see(index, field, view, shard) + //vv("CountRange start=0x%x, endx=0x%x", start, end) defer func() { if r := recover(); r != nil { + c.Dump() AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack()) panic(r) } @@ -397,7 +593,7 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, b, errB := c.b.CountRange(index, field, view, shard, start, end) if a != b { - panic(fmt.Sprintf("a = %v, but b = %v", a, b)) + panic(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b)) } compareErrors(errA, errB) @@ -405,6 +601,7 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, } func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) { + c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack()) @@ -419,3 +616,174 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star compareErrors(errA, errB) return b, errB } + +func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { + c.checker.see(index, field, view, shard) + defer func() { + if r := recover(); r != nil { + c.Dump() + AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack()) + panic(r) + } + }() + + rcA, szA, errA := c.a.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) + rcB, szB, errB := c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) + + compareErrors(errA, errB) + + // We are seeing Roaring vs Badger size differences on + // server/ test TestClusterResize_AddNode/ContinuousShards, + // so turn off the szA vs szB checks and MutliReaderB use. But keep them if we want to + // check RBF vs Badger for byte-for-byte compatiblity (we + // suspect the ops log or optimized bitmaps are accounting for the difference). + sizeMustMatch := false + if sizeMustMatch { + if szA != szB { + panic(fmt.Sprintf("szA(%v) = %v, but szB(%v) = %v; fragmentPathForRoaring='%v'", c.as, szA, c.bs, szB, fragmentPathForRoaring)) + } + return &MultiReaderB{a: rcA, b: rcB}, szB, errB + } else { + // one db won't get data if we do + //return &MultiReaderB{a: rcA, b: rcB, allowSizeVariation: true}, szB, errB + _, _ = szA, errA + rcA.Close() + return rcB, szB, errB + } +} + +func (c *blueGreenTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { + // doesn't change state, so we don't really need see() call here. And we don't have a single shard for it. + //c.checker.see(index, field, view, shard) // don't have shard. + defer func() { + if r := recover(); r != nil { + c.Dump() + AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack()) + panic(r) + } + }() + slcA, errA := c.a.SliceOfShards(index, field, view, optionalViewPath) + slcB, errB := c.b.SliceOfShards(index, field, view, optionalViewPath) + compareErrors(errA, errB) + + // sort order may be different, and that's ok. + cpa := append([]uint64{}, slcA...) + cpb := append([]uint64{}, slcB...) + sort.Slice(cpa, func(i, j int) bool { return cpa[i] < cpa[j] }) + sort.Slice(cpb, func(i, j int) bool { return cpb[i] < cpb[j] }) + + if !reflect.DeepEqual(cpa, cpb) { + // report the first difference + ma := make(map[uint64]bool) + for _, ka := range slcA { + ma[ka] = true + } + for _, kb := range slcB { + if !ma[kb] { + c.Dump() + panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B(%v) had %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb)) + } + delete(ma, kb) + } + if len(ma) != 0 { + for firstDifference := range ma { + panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! A(%v) had %v, but B(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.as, firstDifference, c.bs, cpa, cpb)) + } + } + panic(fmt.Sprintf("blueGreenTx SliceOfShards diference \n slcA(%v)='%#v';\n slcB(%v)='%#v';\n", c.as, cpa, c.bs, cpb)) + } + return slcB, errB +} + +// MultiReaderB is returned by RoaringBitmapReader. It verifies +// that identical byte streams are read from its two members. +type MultiReaderB struct { + a io.ReadCloser + b io.ReadCloser + + allowSizeVariation bool +} + +// Read implements the standard io.Reader method. It panics +// if "a" and "b" have even one byte different in their reads. +func (m *MultiReaderB) Read(p []byte) (nB int, errB error) { + nB, errB = m.b.Read(p) + p2 := make([]byte, nB) + + // read (and discard after comparing for equality) the exact same amount from A. + + // ReadAtLeast reads from r into buf until it has read at least + // min bytes. It returns the number of bytes copied and an error + // if fewer bytes were read. The error is EOF only if no bytes + // were read. If an EOF happens after reading fewer than min bytes, + // ReadAtLeast returns ErrUnexpectedEOF. If min is greater than + // the length of buf, ReadAtLeast returns ErrShortBuffer. On + // return, n >= min if and only if err == nil. If r returns + // an error having read at least min bytes, the error is dropped. + nA, errA := io.ReadAtLeast(m.a, p2, nB) + + if !m.allowSizeVariation { + if errA == io.ErrUnexpectedEOF { + panic(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA)) + } + if nA != nB { + panic(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA)) + } + cmp := bytes.Compare(p[:nB], p2[:nB]) + if cmp != 0 { + panic(fmt.Sprintf("MultiReaderB reads p and p2 (cmp= %v) differed.", cmp)) // \np ='%v'; \np2 ='%v'", cmp, string(p[:nB]), string(p2[:nA]))) + } + } + return +} + +func (m *MultiReaderB) Close() error { + m.a.Close() + return m.b.Close() +} + +// blueGreenChecker is used +type blueGreenChecker struct { + visited map[string]map[string]map[string]map[uint64]struct{} + + // lock mu when using visited. + // otherwise concurrent map writes on TestAPI_Import/RowIDColumnKey + mu sync.Mutex +} + +// see would mark a thing as seen. +func (b *blueGreenChecker) see(index, field, view string, shard uint64) { + // keep this next Printf. Useful to see the sequence of Tx operations. + //fmt.Printf("blueGreenTx.%v\n", Caller(1)) + + b.mu.Lock() + defer b.mu.Unlock() + + if b.visited == nil { + b.visited = make(map[string]map[string]map[string]map[uint64]struct{}) + } + var visitedIdx map[string]map[string]map[uint64]struct{} + var visitedField map[string]map[uint64]struct{} + var visitedView map[uint64]struct{} + + if visitedIdx = b.visited[index]; visitedIdx == nil { + visitedIdx = make(map[string]map[string]map[uint64]struct{}) + b.visited[index] = visitedIdx + } + if visitedField = visitedIdx[field]; visitedField == nil { + visitedField = make(map[string]map[uint64]struct{}) + visitedIdx[field] = visitedField + } + if visitedView = visitedField[view]; visitedView == nil { + visitedView = make(map[uint64]struct{}) + visitedField[view] = visitedView + } + visitedView[shard] = struct{}{} +} + +// seen reports the things it has seen, exactly once so +// that Rollback can be called after Commit without repeating +// the check. +func (b *blueGreenChecker) seen() map[string]map[string]map[string]map[uint64]struct{} { + return b.visited +} diff --git a/bluegreentx_test.go b/bluegreentx_test.go new file mode 100644 index 000000000..131712450 --- /dev/null +++ b/bluegreentx_test.go @@ -0,0 +1,91 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "bytes" + "io" + "io/ioutil" + "testing" + + cryrand "crypto/rand" +) + +func TestMultiReaderB(t *testing.T) { + // MultiReaderB should read identical chunks of bytes from both its "a" and "b" + // member io.Readers, else it should panic. This should hold for + // varying sizes of inputs. + + for n := 1 << 5; n < (1 << 18); n = n*2 - 13 { + src := io.LimitReader(cryrand.Reader, int64(n)) + + a := make([]byte, n) + nr := 0 + for nr < n { + na, err := src.Read(a) + panicOn(err) + nr += na + } + if nr != n { + panic("short read") + } + + b := make([]byte, n) + copy(b, a) + if !bytes.Equal(a, b) { + panic("test prep failed") + } + + m := &MultiReaderB{ + a: ioutil.NopCloser(bytes.NewBuffer(a)), + b: ioutil.NopCloser(bytes.NewBuffer(b)), + } + + // should not trigger the internal panic of MultiReadB + ncp, err := io.Copy(ioutil.Discard, m) + panicOn(err) + if ncp != int64(n) { + panic("short copy") + } + + for victim := 0; victim < n; victim += 7 { + + copy(b, a) + if victim%2 == 0 { + // corrupt b + b[victim] = (b[victim] + 1) % 255 + } else { + // corrupt a + a[victim] = (a[victim] + 1) % 255 + } + m = &MultiReaderB{ + a: ioutil.NopCloser(bytes.NewBuffer(a)), + b: ioutil.NopCloser(bytes.NewBuffer(b)), + } + helperShouldPanicOnCopy(m) + } + } +} + +func helperShouldPanicOnCopy(m *MultiReaderB) { + // differences in bytes read should be noticed + defer func() { + r := recover() + if r == nil { + panic("expected panic on byte difference but didn't see it") + } + }() + _, _ = io.Copy(ioutil.Discard, m) +} diff --git a/catcher.go b/catcher.go index a4d832c20..2394eb331 100644 --- a/catcher.go +++ b/catcher.go @@ -16,6 +16,7 @@ package pilosa import ( "fmt" + "io" "github.com/pilosa/pilosa/v2/roaring" ) @@ -51,14 +52,18 @@ func (c *catcherTx) WholeDatabaseBlake3Hash(index, field, view string, shard uin return c.b.WholeDatabaseBlake3Hash(index, field, view, shard) } -func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { +func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { defer func() { if r := recover(); r != nil { AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack()) panic(r) } }() - return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize) + return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) +} + +func (c *catcherTx) Dump() { + c.b.Dump() } func (c *catcherTx) Readonly() bool { @@ -275,3 +280,26 @@ func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, }() return c.b.OffsetRange(index, field, view, shard, offset, start, end) } + +func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { + defer func() { + if r := recover(); r != nil { + AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack()) + panic(r) + } + }() + return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) +} + +func (c *catcherTx) Type() string { + return c.b.Type() +} +func (c *catcherTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { + defer func() { + if r := recover(); r != nil { + AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack()) + panic(r) + } + }() + return c.b.SliceOfShards(index, field, view, optionalViewPath) +} diff --git a/cluster.go b/cluster.go index 479a5c6f2..eb0cba364 100644 --- a/cluster.go +++ b/cluster.go @@ -110,6 +110,17 @@ func (a Nodes) ContainsID(id string) bool { return false } +// NodeByID returns the node for an ID. If the ID is not found, +// it returns nil. +func (a Nodes) NodeByID(id string) *Node { + for _, n := range a { + if n.ID == id { + return n + } + } + return nil +} + // Filter returns a new list of nodes with node removed. func (a Nodes) Filter(n *Node) []*Node { other := make([]*Node, 0, len(a)) @@ -514,7 +525,7 @@ func (c *cluster) unprotectedSetState(state string) { cleaner.Cluster = c cleaner.Closing = c.closing - // Clean holder. + // Clean holder. This is where the shard gets removed after resize. if err := cleaner.CleanHolder(); err != nil { c.logger.Printf("holder clean error: err=%s", err) } @@ -1027,20 +1038,49 @@ func (c *cluster) ownsShard(nodeID string, index string, shard uint64) bool { func (c *cluster) partitionNodes(partitionID int) []*Node { // Default replica count to between one and the number of nodes. // The replica count can be zero if there are no nodes. + + // Assume that c.nodes may be missing a node that is part of the cluster but not currently present. + // The partition calculation must use the full cluster size in BOTH cases: + // - use len(c.Topology.nodeIDs) instead of len(c.nodes), + // - collect nodes from c.Topology.nodeIDs rather than from c.nodes, + // - when the node is missing, it should be considered, found absent from c.nodes, then omitted from the return slice. + + // Use c.Topology to determine cluster membership when it + // exists and contains data. Otherwise, fall back to using + // c.nodes. The only time c.Topology should be nil is in + // tests. + var useTopology bool + if c.Topology != nil && len(c.Topology.nodeIDs) > 0 { + useTopology = true + } + replicaN := c.ReplicaN - if replicaN > len(c.nodes) { - replicaN = len(c.nodes) + var nodeN int + if useTopology { + nodeN = len(c.Topology.nodeIDs) + } else { + nodeN = len(c.nodes) + } + if replicaN > nodeN { + replicaN = nodeN } else if replicaN == 0 { replicaN = 1 } // Determine primary owner node. - nodeIndex := c.Hasher.Hash(uint64(partitionID), len(c.nodes)) + nodeIndex := c.Hasher.Hash(uint64(partitionID), nodeN) // Collect nodes around the ring. - nodes := make([]*Node, replicaN) + nodes := make([]*Node, 0, replicaN) for i := 0; i < replicaN; i++ { - nodes[i] = c.nodes[(nodeIndex+i)%len(c.nodes)] + if useTopology { + maybeNodeID := c.Topology.nodeIDs[(nodeIndex+i)%nodeN] + if node := Nodes(c.nodes).NodeByID(maybeNodeID); node != nil { + nodes = append(nodes, node) + } + } else { + nodes = append(nodes, c.nodes[(nodeIndex+i)%len(c.nodes)]) + } } return nodes @@ -2193,7 +2233,7 @@ func (c *cluster) nodeStatus() *NodeStatus { func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error { c.mu.Lock() defer c.mu.Unlock() - c.logger.Printf("merge cluster status: node=%s cluster=%v", c.Node.ID, cs) + c.logger.Printf("merge cluster status: node=%s cluster=%v, topologySize=%v", c.Node.ID, cs, len(c.Topology.nodeIDs)) // Ignore status updates from self (coordinator). if c.unprotectedIsCoordinator() { return nil diff --git a/cluster_internal_test.go b/cluster_internal_test.go index c31b9f294..32ceec7f7 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -96,7 +96,9 @@ func newIndexWithTempPath(name string) *Index { if err != nil { panic(err) } - index, err := NewIndex(NewHolder(DefaultPartitionN), path, name) + h := NewHolder(DefaultPartitionN) + h.Path = path + index, err := h.CreateIndex(name, IndexOptions{}) if err != nil { panic(err) } @@ -160,7 +162,7 @@ func TestFragSources(t *testing.T) { defer idx.Close() // Obtain transaction. - tx := &RoaringTx{Index: idx} + tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx}) defer tx.Rollback() field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault()) @@ -809,12 +811,12 @@ func TestCluster_ResizeStates(t *testing.T) { if idx0 == nil { t.Fatal(`idx0 was nil, could not retrieve Index("i")`) } - //idx0.Dump("node0") // addNode needs to block until the resize process has completed. if err := tc.addNode(); err != nil { t.Fatalf("adding node: %v", err) } + node1 := tc.Clusters[1] // Ensure that nodes come up in state NORMAL. @@ -846,15 +848,12 @@ func TestCluster_ResizeStates(t *testing.T) { if idx1 == nil { t.Fatal(`idx1 was nil, could not retrieve Index("i")`) } - //idx0.Dump("after rebalance, node0") - //idx1.Dump("after rebalance, node1") // Ensure checksums are the same. if chksum, err := node1Fragment.Checksum(); err != nil { t.Fatal(err) } else if !bytes.Equal(chksum, node0Checksum) { t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum) - // badger red: TestCluster_ResizeStates/Multiple_nodes,_with_data: cluster_internal_test.go:841: expected standard view checksum to match: ef46db3751d8e999 - fad4de25ee696ca0 } // Close TestCluster. @@ -872,6 +871,7 @@ func TestAE(t *testing.T) { c.abortAntiEntropy() close(ch) }() + defer c.abortAntiEntropyQ() // avoid leaking a goroutine. select { case <-ch: return @@ -883,11 +883,13 @@ func TestAE(t *testing.T) { t.Run("AbortBlocksInitialized", func(t *testing.T) { c := newCluster() c.initializeAntiEntropy() + ch := make(chan struct{}) go func() { c.abortAntiEntropy() close(ch) }() + defer c.abortAntiEntropyQ() // avoid leak of goroutine. select { case <-ch: t.Fatalf("aborting anti entropy on an initialized cluster didn't block") diff --git a/cmd/loader/loader.go b/cmd/loader/loader.go new file mode 100644 index 000000000..2a6a00ff2 --- /dev/null +++ b/cmd/loader/loader.go @@ -0,0 +1,150 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "archive/tar" + "compress/gzip" + "context" + "time" + //"fmt" + "fmt" + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/http" + "io" + "io/ioutil" + gohttp "net/http" + //"log" + "os" + //"path/filepath" + //"sort" + "strconv" + "strings" +) + +func UploadTar(srcFile string, client *http.InternalClient) error { + t0 := time.Now() + + f, err := os.Open(srcFile) + if err != nil { + return (err) + } + defer f.Close() + var tarReader *tar.Reader + if strings.HasSuffix(srcFile, "gz") { + gzf, err := gzip.NewReader(f) + if err != nil { + return err + } + tarReader = tar.NewReader(gzf) + } else { + tarReader = tar.NewReader(f) + } + viewData := make(map[string][]byte) + //given ordered by index/field/view + //trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255 + lastIndex := "" + lastField := "" + lastShard := uint64(0) + //vv("top of tar loop") + n := 0 + for { + header, err := tarReader.Next() + if err == io.EOF { + if header != nil { + panic("header should not be nil on err io.EOF") + } + //submit any stuff we have left + if len(viewData) > 0 { + request := &pilosa.ImportRoaringRequest{ + Views: viewData, + } + // Submit(lastIndex, lastField, lastShard, request) + //vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard) + uri := GetImportRoaringURI(lastIndex, lastShard) + err := client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request) + panicOn(err) + //vv("done with submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard) + } + return nil + } + //vv("got header '%v'", header.Name) + n++ + if n%500 == 0 { + vv("n = %v, progress, elapsed '%v'", n, time.Since(t0)) + } + parts := strings.Split(header.Name, "/") + index := parts[0] + field := parts[1] + view := parts[3] + shard, err := strconv.ParseUint(parts[5], 10, 64) + if err != nil { + return err + } + // TODO: shards can be loaded in parallel, so maybe farm out to a worker set of goro. + if index != lastIndex || field != lastField || shard != lastShard { + if len(viewData) > 0 { + request := &pilosa.ImportRoaringRequest{ + Views: viewData, + } + //vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard) + uri := GetImportRoaringURI(lastIndex, lastShard) + panicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)) + viewData = make(map[string][]byte) + //vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0)) + + } + } + roaringData, err := ioutil.ReadAll(tarReader) + if err != nil { + return err + } + if _, already := viewData[view]; already { + panic(fmt.Sprintf("view '%v' already present!", view)) + } + viewData[view] = roaringData + lastIndex = index + lastField = field + + //lastShard = shard + //vv("bottom of loop") + } +} + +func main() { + + host := "127.0.0.1:10101" + h := &gohttp.Client{} + c, err := http.NewInternalClient(host, h) + panicOn(err) + + tarSrcPath := "q2.tar.gz" + t0 := time.Now() + panicOn(UploadTar(tarSrcPath, c)) + vv("total elapsed '%v'", time.Since(t0)) +} + +var globURI *pilosa.URI + +func init() { + var err error + globURI, err = pilosa.NewURIFromHostPort("127.0.0.1", 10101) + panicOn(err) +} + +// get correct node to go to. +func GetImportRoaringURI(index string, shard uint64) *pilosa.URI { + return globURI +} diff --git a/cmd/loader/vprint.go b/cmd/loader/vprint.go new file mode 100644 index 000000000..e6bea8d55 --- /dev/null +++ b/cmd/loader/vprint.go @@ -0,0 +1,177 @@ +// home: https://github.com/glyerine/vprint +// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved. +// License: MIT +// +// MIT License +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package main + +import ( + "fmt" + "io" + "os" + "path" + "runtime" + "runtime/debug" + "sync" + "time" +) + +const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00" +const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00" + +// for tons of debug output +var VerboseVerbose bool = false + +// convience functions for . import +var pp = PP +var vv = VV + +var panicOn = PanicOn + +func init() { + // keeper linter happy + _ = pp + _ = vv +} + +func PanicOn(err error) { + if err != nil { + panic(err) + } +} + +func PP(format string, a ...interface{}) { + if VerboseVerbose { + TSPrintf(format, a...) + } +} + +func VV(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +func AlwaysPrintf(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +var tsPrintfMut sync.Mutex + +// time-stamped printf +func TSPrintf(format string, a ...interface{}) { + tsPrintfMut.Lock() + Printf("\n%s %s ", FileLine(3), ts()) + Printf(format+"\n", a...) + tsPrintfMut.Unlock() +} + +// get timestamp for logging purposes +func ts() string { + return time.Now().Format(RFC3339UsecTz0) +} + +// so we can multi write easily, use our own printf +var OurStdout io.Writer = os.Stdout + +// Printf formats according to a format specifier and writes to standard output. +// It returns the number of bytes written and any write error encountered. +func Printf(format string, a ...interface{}) (n int, err error) { + return fmt.Fprintf(OurStdout, format, a...) +} + +func FileLine(depth int) string { + _, fileName, fileLine, ok := runtime.Caller(depth) + var s string + if ok { + s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine) + } else { + s = "" + } + return s +} + +func stack() string { + return string(debug.Stack()) +} + +func FileExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return false + } + return true +} + +func DirExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return true + } + return false +} + +func FileSize(name string) (int64, error) { + fi, err := os.Stat(name) + if err != nil { + return -1, err + } + return fi.Size(), nil +} + +// Caller returns the name of the calling function. +func Caller(upStack int) string { + // elide ourself and runtime.Callers + target := upStack + 2 + + pc := make([]uintptr, target+2) + n := runtime.Callers(0, pc) + + f := runtime.Frame{Function: "unknown"} + if n > 0 { + frames := runtime.CallersFrames(pc[:n]) + for i := 0; i <= target; i++ { + contender, more := frames.Next() + if i == target { + f = contender + } + if !more { + break + } + } + } + return f.Function +} + +// happy linter: +var _ = DirExists +var _ = FileExists +var _ = Caller +var _ = stack +var _ = RFC3339MsecTz0 +var _ = RFC3339UsecTz0 +var _ = AlwaysPrintf +var _ = FileSize diff --git a/executor.go b/executor.go index 62b98b5be..6a072372a 100644 --- a/executor.go +++ b/executor.go @@ -19,12 +19,12 @@ import ( "encoding/json" "fmt" "math" + "math/bits" "sort" "strings" "sync" "time" - "github.com/molecula/ext" "github.com/pilosa/pilosa/v2/pql" pb "github.com/pilosa/pilosa/v2/proto" "github.com/pilosa/pilosa/v2/roaring" @@ -64,12 +64,6 @@ type executor struct { workersWG sync.WaitGroup workerPoolSize int work chan job - // global registry to check for name clashes - additionalOps map[string]*ext.BitmapOp - // typed registries we can use in lookups - additionalBitmapOps map[string]ext.BitmapOpBitmap - additionalCountOps map[string]ext.BitmapOpUnaryCount - additionalFieldOps map[string]ext.BitmapOpBSIBitmap } // executorOption is a functional option type for pilosa.Executor @@ -125,38 +119,6 @@ func (e *executor) Close() error { return nil } -func (e *executor) registerOps(ops []ext.BitmapOp) error { - if e.additionalOps == nil { - e.additionalOps = make(map[string]*ext.BitmapOp) - e.additionalBitmapOps = make(map[string]ext.BitmapOpBitmap) - e.additionalCountOps = make(map[string]ext.BitmapOpUnaryCount) - e.additionalFieldOps = make(map[string]ext.BitmapOpBSIBitmap) - } - for i, op := range ops { - name := op.Name - if _, exists := e.additionalOps[name]; exists { - return fmt.Errorf("op name '%s' already defined", name) - } - e.additionalOps[name] = &ops[i] - typ := ops[i].Func.BitmapOpType() - switch { - case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount: - e.additionalCountOps[name] = ops[i].Func.(ext.BitmapOpUnaryCount) - case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap: - e.additionalBitmapOps[name] = ops[i].Func.(ext.BitmapOpBitmap) - case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap: - if fn, ok := ops[i].Func.(ext.BitmapOpBSIBitmapPrecall); ok { - e.additionalFieldOps[name] = ext.BitmapOpBSIBitmap(fn) - } else { - e.additionalFieldOps[name] = ops[i].Func.(ext.BitmapOpBSIBitmap) - } - default: - return fmt.Errorf("unsupported types for '%s': input type %d, output type %d", name, typ.Input, typ.Output) - } - } - return nil -} - // Execute executes a PQL query. func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute") @@ -228,7 +190,6 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar } else if err := validateQueryContext(ctx); err != nil { return resp, err } - resp.Results = results // Fill column attributes if requested. @@ -369,7 +330,6 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr // handlePreCalls traverses the call tree looking for calls that need // precomputed values. Right now, that's just Distinct. func (e *executor) handlePreCalls(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) error { - if c.Name == "Precomputed" { idx := c.Args["valueidx"].(int64) if idx >= 0 && idx < int64(len(opt.EmbeddedData)) { @@ -404,18 +364,17 @@ func (e *executor) handlePreCalls(ctx context.Context, tx Tx, index string, c *p // like Distinct, where you can't predict output shard for a result // from the shard being queried. if newIndex != "" && newIndex != index { - if err := e.handlePreCallChildren(ctx, tx, index, c, shards, opt); err != nil { - return err - } - c.Type = pql.PrecallGlobal index = newIndex // we need to recompute shards, then shards = nil } + if err := e.handlePreCallChildren(ctx, tx, index, c, shards, opt); err != nil { + return err + } + // child calls already handled, no precall for this, so we're done if c.Type == pql.PrecallNone { - // otherwise, handle the children - return e.handlePreCallChildren(ctx, tx, index, c, shards, opt) + return nil } // We don't try to handle sub-calls from here. I'm not 100% // sure that's right, but I think the fact that they're happening @@ -548,6 +507,7 @@ func (e *executor) execute(ctx context.Context, tx Tx, index string, q *pql.Quer if err != nil { return nil, err } + results = append(results, v) // Some Calls can have significant data associated with them // that gets generated during processing, such as Precomputed @@ -708,15 +668,6 @@ func (e *executor) executeCall(ctx context.Context, tx Tx, index string, c *pql. return nil, err } - // Special handling for mutation and top-n calls. - if op, ok := e.additionalCountOps[c.Name]; ok { - statFn() - return e.executeGenericCount(ctx, tx, index, c, op, shards, opt) - } - if op, ok := e.additionalFieldOps[c.Name]; ok { - statFn() - return e.executeGenericField(ctx, tx, index, c, op, shards, opt) - } switch c.Name { case "Sum": statFn() @@ -739,6 +690,9 @@ func (e *executor) executeCall(ctx context.Context, tx Tx, index string, c *pql. case "ClearRow": statFn() return e.executeClearRow(ctx, tx, index, c, shards, opt) + case "Distinct": + statFn() + return e.executeDistinct(ctx, tx, index, c, shards, opt) case "Store": statFn() return e.executeSetRow(ctx, tx, index, c, shards, opt) @@ -1151,11 +1105,9 @@ func (e *executor) executeSum(ctx context.Context, tx Tx, index string, c *pql.C return other, nil } -// executeGenericField executes a generic call on a field. Note that in this -// implementation, the operation is always a BSI op. -func (e *executor) executeGenericField(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shards []uint64, opt *execOptions) (SignedRow, error) { - span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericField") - span.LogKV("name", c.Name) +// executeDistinct executes a Distinct call on a field. +func (e *executor) executeDistinct(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (SignedRow, error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinct") defer span.Finish() field := c.Args["field"] @@ -1165,7 +1117,7 @@ func (e *executor) executeGenericField(ctx context.Context, tx Tx, index string, // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - return e.executeGenericFieldShard(ctx, tx, index, c, op, shard) + return e.executeDistinctShard(ctx, tx, index, c, shard) } // Merge returned results at coordinating node. @@ -1440,13 +1392,6 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index stri span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard") defer span.Finish() - if _, ok := e.additionalCountOps[c.Name]; ok { - return nil, fmt.Errorf("count op %s used as bitmap call", c.Name) - } - if op, ok := e.additionalBitmapOps[c.Name]; ok { - return e.executeGenericBitmapShard(ctx, tx, index, c, op, shard) - } - switch c.Name { case "Row", "Range": return e.executeRowShard(ctx, tx, index, c, shard) @@ -1464,6 +1409,8 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index stri return e.executeShiftShard(ctx, tx, index, c, shard) case "All": // Allow a shard computation to use All() (note, limit/offset not applied) return e.executeAllCallShard(ctx, tx, index, c, shard) + case "Distinct": + return nil, errors.New("Distinct shouldn't be hit as a bitmap call") case "Precomputed": return e.executePrecomputedCallShard(ctx, tx, index, c, shard) default: @@ -1471,11 +1418,10 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index stri } } -// executeGenericFieldShard executes a generic/extension command on a -// single shard. Note that in this implementation, the op is always -// a BSI op. -func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shard uint64) (SignedRow, error) { - span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericShard") +// executeDistinctShard executes a Distinct call on a single shard, yielding +// a SignedRow of the values found. +func (e *executor) executeDistinctShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (result SignedRow, err error) { + span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinctShard") defer span.Finish() var filter *Row @@ -1483,7 +1429,7 @@ func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index st if len(c.Children) == 1 { row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) if err != nil { - return SignedRow{}, errors.Wrap(err, "executing bitmap call") + return result, errors.Wrap(err, "executing bitmap call") } filter = row if filter != nil && len(filter.segments) > 0 { @@ -1497,29 +1443,115 @@ func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index st field := e.Holder.Field(index, fieldName) if field == nil { - return SignedRow{}, nil + return result, nil } bsig := field.bsiGroup(fieldName) if bsig == nil { - return SignedRow{}, nil + return result, nil + } + view := viewBSIGroupPrefix + fieldName + + depth := uint64(bsig.BitDepth) + offset := bsig.Base + + existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*0, ShardWidth*1) + if err != nil { + return result, err + } + if filter != nil { + existsBitmap = existsBitmap.Intersect(filterBitmap) + } + if !existsBitmap.Any() { + return result, nil } - fragment := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard) - if fragment == nil { - return SignedRow{}, nil + signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*1, ShardWidth*2) + if err != nil { + return result, nil } - var out ext.SignedBitmap - if filterBitmap != nil { - out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{WrapBitmap(filterBitmap)}, c.Args) - } else { - out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{}, c.Args) + dataBitmaps := make([]*roaring.Bitmap, depth) + + for i := uint64(0); i < depth; i++ { + dataBitmaps[i], err = tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*(i+2), ShardWidth*(i+3)) + if err != nil { + return result, err + } } + // we need spaces for sign bit, existence/filter bit, and data + // row bits, which we'll be grabbing 65k bits at a time + stashWords := make([]uint64, 1024*(depth+2)) + bitStashes := make([][]uint64, depth) + for i := uint64(0); i < depth; i++ { + start := i * 1024 + last := start + 1024 + bitStashes[i] = stashWords[start:last] + i++ + } + stashOffset := depth * 1024 + existStash := stashWords[stashOffset : stashOffset+1024] + signStash := stashWords[stashOffset+1024 : stashOffset+2048] + dataBits := make([][]uint64, depth) + + posValues := make([]uint64, 0, 64) + negValues := make([]uint64, 0, 64) + + posBitmap := roaring.NewFileBitmap() + negBitmap := roaring.NewFileBitmap() + + existIterator, _ := existsBitmap.Containers.Iterator(0) + for existIterator.Next() { + key, value := existIterator.Value() + if value.N() == 0 { + continue + } + exists := value.AsBitmap(existStash) + sign := signBitmap.Containers.Get(key).AsBitmap(signStash) + for i := uint64(0); i < depth; i++ { + dataBits[i] = dataBitmaps[i].Containers.Get(key).AsBitmap(bitStashes[i]) + } + for idx, word := range exists { + // mask holds a mask we can test the other words against. + mask := uint64(1) + for word != 0 { + shift := uint(bits.TrailingZeros64(word)) + // we shift one *more* than that, to move the + // actual one bit off. + word >>= shift + 1 + mask <<= shift + value := int64(0) + for b := uint64(0); b < depth; b++ { + if dataBits[b][idx]&mask != 0 { + value += (1 << b) + } + } + if sign[idx]&mask != 0 { + value *= -1 + } + value += int64(offset) + if value < 0 { + negValues = append(negValues, uint64(-value)) + } else { + posValues = append(posValues, uint64(value)) + } + // and now we processed that bit, so we move the mask over one. + mask <<= 1 + } + if len(negValues) > 0 { + _, _ = negBitmap.AddN(negValues...) + negValues = negValues[:0] + } + if len(posValues) > 0 { + _, _ = posBitmap.AddN(posValues...) + posValues = posValues[:0] + } + } + } return SignedRow{ - Neg: NewRowFromBitmap(UnwrapBitmap(out.Neg)), - Pos: NewRowFromBitmap(UnwrapBitmap(out.Pos)), + Neg: NewRowFromBitmap(negBitmap), + Pos: NewRowFromBitmap(posBitmap), }, nil } @@ -2681,11 +2713,13 @@ func (e *executor) executeRowsShard(ctx context.Context, tx Tx, index string, fi } func (e *executor) executeRowShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowShard") defer span.Finish() // Handle bsiGroup ranges differently. if c.HasConditionArg() { + // looks the same on badger/roaring. we think. return e.executeRowBSIGroupShard(ctx, tx, index, c, shard) } @@ -2795,6 +2829,7 @@ func (e *executor) executeRowShard(ctx context.Context, tx Tx, index string, c * // executeRowBSIGroupShard executes a range(bsiGroup) call for a local shard. func (e *executor) executeRowBSIGroupShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (_ *Row, err error) { + span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowBSIGroupShard") defer span.Finish() @@ -2975,44 +3010,6 @@ func (e *executor) executeIntersectShard(ctx context.Context, tx Tx, index strin return other, nil } -// executeGenericBitmapShard executes a generic bitmap call for a local shard. -func (e *executor) executeGenericBitmapShard(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBitmap, shard uint64) (*Row, error) { - span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericBitmapShard") - defer span.Finish() - - if op.BitmapOpArity() == ext.OpArityUnary { - if len(c.Children) != 1 { - return nil, fmt.Errorf("%s needs exactly one row parameter", c.Name) - } - row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) - if err != nil { - return nil, err - } - return row.GenericUnaryOp(op.BitmapOpFunc(), c.Args), nil - } - - var err error - rows := make([]*Row, len(c.Children)) - for i, input := range c.Children { - rows[i], err = e.executeBitmapCallShard(ctx, tx, index, input, shard) - if err != nil { - return nil, err - } - } - var other *Row - switch op.BitmapOpArity() { - case ext.OpArityBinary: - other = rows[0] - for _, row := range rows[1:] { - other = other.GenericBinaryOp(op.BitmapOpFunc(), row, c.Args) - } - case ext.OpArityNary: - other = rows[0].GenericNaryOp(op.BitmapOpFunc(), rows[1:], c.Args) - } - other.invalidateCount() - return other, nil -} - // executeUnionShard executes a union() call for a local shard. func (e *executor) executeUnionShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard") @@ -3164,41 +3161,6 @@ func (e *executor) executeShiftShard(ctx context.Context, tx Tx, index string, c return row.Shift(n) } -// executeGeneric executes a provided count-like call. -func (e *executor) executeGenericCount(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpUnaryCount, shards []uint64, opt *execOptions) (uint64, error) { - span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericCount") - defer span.Finish() - - if len(c.Children) == 0 { - return 0, fmt.Errorf("%s() requires an input bitmap", c.Name) - } else if len(c.Children) > 1 { - return 0, fmt.Errorf("%s() only accepts a single bitmap input", c.Name) - } - - // Execute calls in bulk on each remote node and merge. - mapFn := func(ctx context.Context, shard uint64) (interface{}, error) { - row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard) - if err != nil { - return 0, err - } - return row.GenericCount(op, c.Args), nil - } - - // Merge returned results at coordinating node. - reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { - other, _ := prev.(uint64) - return other + v.(uint64) - } - - result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn) - if err != nil { - return 0, err - } - n, _ := result.(uint64) - - return n, nil -} - // executeCount executes a count() call. func (e *executor) executeCount(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount") diff --git a/executor_test.go b/executor_test.go index 9a78b96c0..6fd5a7b61 100644 --- a/executor_test.go +++ b/executor_test.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "math" "math/rand" + "os" "reflect" "strconv" "strings" @@ -513,6 +514,12 @@ func TestExecutor_Execute_Count(t *testing.T) { } +func roaringOnlyTest(t *testing.T) { + if os.Getenv("PILOSA_TXSRC") != "roaring" { + t.Skip("skip for everything but roaring") + } +} + // Ensure a set query can be executed. func TestExecutor_Execute_Set(t *testing.T) { t.Run("RowIDColumnID", func(t *testing.T) { @@ -521,7 +528,7 @@ func TestExecutor_Execute_Set(t *testing.T) { cmd := cluster[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} - hldr.SetBit("i", "f", 1, 0) + hldr.SetBit("i", "f", 1, 0) // creates and commits a Tx internally. t.Run("OK", func(t *testing.T) { hldr.ClearBit("i", "f", 11, 1) @@ -582,10 +589,10 @@ func TestExecutor_Execute_Set(t *testing.T) { cmd := cluster[0] holder := cmd.Server.Holder() hldr := test.Holder{Holder: holder} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) + idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true}) t.Run("OK", func(t *testing.T) { - hldr.SetBit("i", "f", 1, 0) + hldr.SetBit("i", "f", 1, 0) // creates and Commits a Tx internally. if n := hldr.Row("i", "f", 11).Count(); n != 0 { t.Fatalf("unexpected row count: %d", n) } @@ -619,14 +626,16 @@ func TestExecutor_Execute_Set(t *testing.T) { }) t.Run("ErrInvalidColValueType", func(t *testing.T) { - if err := index.DeleteField("f"); err != nil { - t.Fatal(err) - } - if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + + if err := idx.DeleteField("f"); err != nil { t.Fatal(err) } - if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(2.1, f=1)`}); err == nil || strings.Contains(err.Error(), `column value must be a string or non-negative integer`) { + if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil { + t.Fatal(err) + } + + if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(2.1, f=1)`}); err == nil || !strings.Contains(err.Error(), "parse error") { t.Fatal(err) } @@ -637,9 +646,9 @@ func TestExecutor_Execute_Set(t *testing.T) { } }) - t.Run("ErrInvalidRowValueType", func(t *testing.T) { - index := hldr.MustCreateIndexIfNotExists("inokey", pilosa.IndexOptions{}) - if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { + t.Run("ErrInvalidRowValueType", func(t *testing.T) { // // failing under badger_roaring + idx := hldr.MustCreateIndexIfNotExists("inokey", pilosa.IndexOptions{}) + if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil { t.Fatal(err) } if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "inokey", Query: `Set(2, f=1.2)`}); err == nil || !strings.Contains(err.Error(), "row value must be a string or non-negative integer") { @@ -3413,6 +3422,7 @@ func TestExecutor_Execute_Existence(t *testing.T) { defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatal(err) @@ -3438,6 +3448,7 @@ func TestExecutor_Execute_Existence(t *testing.T) { } else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) { t.Fatalf("unexpected columns after Not: %+v", bits) } + // Reopen cluster to ensure existence field is reloaded. if err := c[0].Reopen(); err != nil { t.Fatal(err) @@ -3622,6 +3633,8 @@ func TestExecutor_Execute_FieldValue(t *testing.T) { // Ensure an all query can be executed. func TestExecutor_Execute_All(t *testing.T) { + skipForRBF(t) + t.Run("ColumnID", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -4024,7 +4037,6 @@ func TestExecutor_Execute_ClearRow(t *testing.T) { // Ensure a row can be set. func TestExecutor_Execute_SetRow(t *testing.T) { - t.Run("Set_NewRow", func(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -4084,8 +4096,8 @@ func TestExecutor_Execute_SetRow(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()) + idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) + _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()) if err != nil { t.Fatal(err) } @@ -4174,25 +4186,6 @@ func TestExecutor_Execute_SetRow(t *testing.T) { t.Fatalf("unexpected columns: %+v", bits) } }) - t.Run("Err_Store(Distinct)", func(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c[0].Server.Holder()} - index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true}) - f1, err := index.CreateField("f1", pilosa.OptFieldTypeDefault()) - if err != nil { - t.Fatal(err) - } - f2, err := index.CreateField("f2", pilosa.OptFieldTypeDefault()) - if err != nil { - t.Fatal(err) - } - - q := fmt.Sprintf(`Store(Distinct(field=%s), %s=2)`, f1.Name(), f2.Name()) - if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: index.Name(), Query: q}); err == nil { - t.Fatalf("expected 'unsupported result type' error, got: %+v", res) - } - }) } func benchmarkExistence(nn bool, b *testing.B) { @@ -4402,6 +4395,8 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { } func TestExecutor_GroupByStrings(t *testing.T) { + skipForRBF(t) + c := test.MustRunCluster(t, 1) defer c.Close() c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "generals", pilosa.OptFieldKeys()) @@ -4858,6 +4853,8 @@ func sameStringSlice(x, y []string) bool { } func TestExecutor_Execute_GroupBy(t *testing.T) { + skipForRBF(t) + groupByTest := func(t *testing.T, clusterSize int) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -5312,6 +5309,7 @@ func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOpt defer c.Close() hldr := test.Holder{Holder: c[0].Server.Holder()} index := hldr.MustCreateIndexIfNotExists("i", *indexOptions) + defer index.Close() _, err := index.CreateField("f", fieldOption...) if err != nil { t.Fatal(err) @@ -5723,12 +5721,15 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) { if err := json.NewDecoder(bytes.NewReader(data)).Decode(schema); err != nil { t.Fatal(err) } + if err := api.ApplySchema(context.TODO(), schema, false); err != nil { t.Fatal(err) } + // AntitodePoint == row 1 b/c keys field. writeQuery := `Set(100, type=AntidotePoint)Set(100, equip_id=100)Set(100, site_id=100)Set(100, id=100)` - for _, i := range schema.Indexes { + for k, i := range schema.Indexes { + _ = k if _, err := api.Query(context.TODO(), &pilosa.QueryRequest{Index: i.Name, Query: writeQuery}); err != nil { t.Fatal(err) } @@ -5739,11 +5740,11 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) { Intersect( Distinct( Intersect(Row(type=AntidotePoint)), - index=power_ts, field=equip_id), + index=equipment, field=equip_id), Distinct( Intersect(Row(type=AntidotePoint)), - index=power_ts, field=equip_id) - ), index=equipment, field=site_id)` + index=sites, field=equip_id) + ), index=power_ts, field=site_id)` // Check if test query gives correct results (one column 100) t.Run("Distinct", func(t *testing.T) { @@ -5760,6 +5761,7 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) { } if r.Pos.Count() != 1 { t.Fatalf("invalid pilosa.SignedRow.Pos.Count, expected: 1, got: %v", r.Pos.Count()) + } if r.Pos.Columns()[0] != 100 { t.Fatalf("invalid pilosa.SignedRow.Pos.Columns, expected: [100], got: %v", r.Pos.Columns()) diff --git a/extension.go b/extension.go deleted file mode 100644 index 1a492cce3..000000000 --- a/extension.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2019 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "fmt" - - "github.com/molecula/ext" - "github.com/pilosa/pilosa/v2/roaring" -) - -// WrapBitmap yields an extension-Bitmap from a roaring Bitmap. -func WrapBitmap(bm *roaring.Bitmap) ext.Bitmap { - return wrappedBitmap{bm} -} - -// wrappedBitmap is a very shallow glue shim to convert a roaring Bitmap to -// an extension Bitmap. -type wrappedBitmap struct{ *roaring.Bitmap } - -// UnwrapBitmap converts an extension-bitmap to its underlying roaring Bitmap. -func UnwrapBitmap(bm ext.Bitmap) *roaring.Bitmap { - if inner, ok := bm.(wrappedBitmap); ok { - if inner.Bitmap != nil { - return inner.Bitmap - } - return roaring.NewFileBitmap() - } - return roaring.NewFileBitmap() -} - -func (b wrappedBitmap) Intersect(other ext.Bitmap) ext.Bitmap { - return wrappedBitmap{b.Bitmap.Intersect(other.(wrappedBitmap).Bitmap)} -} - -func (b wrappedBitmap) Union(other ext.Bitmap) ext.Bitmap { - return wrappedBitmap{b.Bitmap.Union(other.(wrappedBitmap).Bitmap)} -} - -func (b wrappedBitmap) IntersectionCount(other ext.Bitmap) uint64 { - return b.Bitmap.IntersectionCount(other.(wrappedBitmap).Bitmap) -} - -func (b wrappedBitmap) Difference(other ext.Bitmap) ext.Bitmap { - return wrappedBitmap{b.Bitmap.Difference(other.(wrappedBitmap).Bitmap)} -} - -func (b wrappedBitmap) Xor(other ext.Bitmap) ext.Bitmap { - return wrappedBitmap{b.Bitmap.Xor(other.(wrappedBitmap).Bitmap)} -} - -func (b wrappedBitmap) Shift(n int) (ext.Bitmap, error) { - shifted, err := b.Bitmap.Shift(n) - return wrappedBitmap{shifted}, err -} - -func (b wrappedBitmap) Flip(start, last uint64) ext.Bitmap { - return wrappedBitmap{b.Bitmap.Flip(start, last)} -} - -func (b wrappedBitmap) New() ext.Bitmap { - return WrapBitmap(roaring.NewFileBitmap()) -} - -// ContainerBits tries to get one container's worth of bits. -func (b wrappedBitmap) ContainerBits(offset uint64, target []uint64) (out []uint64) { - // it's an error to call this with a non-container-aligned offset - if offset&0xFFFF != 0 { - return nil - } - if b.Bitmap == nil { - fmt.Printf("ContainerBits on bitmap with no contents\n") - return nil - } - if b.Bitmap.Containers == nil { - fmt.Printf("ContainerBits on bitmap with nil Containers\n") - return nil - } - c := b.Bitmap.Containers.Get(offset >> 16) - if c == nil { - return nil - } - return c.AsBitmap(target) -} diff --git a/field.go b/field.go index a9a80e2fe..0802c3848 100644 --- a/field.go +++ b/field.go @@ -92,6 +92,8 @@ type Field struct { name string qualifiedName string + idx *Index + viewMap map[string]*view // Row attribute storage and cache @@ -1181,6 +1183,7 @@ func (f *Field) createViewIfNotExistsBase(name string) (*view, bool, error) { func (f *Field) newView(path, name string) *view { view := newView(f.holder, path, f.index, f.name, name, f.options) + view.idx = f.idx view.rowAttrStore = f.rowAttrStore view.stats = f.Stats view.broadcaster = f.broadcaster @@ -1432,6 +1435,14 @@ func (f *Field) SetValue(tx Tx, columnID uint64, value int64) (changed bool, err if err != nil { return false, errors.Wrap(err, "creating view") } + if view.holder == nil { + panic("view.holder should not be nil") + } + if view.idx == nil { + panic("view.idx should not be nil") + } + view.holder.addIndexFromField(view.idx) + return view.setValue(tx, columnID, bsig.BitDepth, baseValue) } @@ -1761,6 +1772,10 @@ func (f *Field) importRoaring(ctx context.Context, tx Tx, data []byte, shard uin return nil } +func (f *Field) GetIndex() *Index { + return f.idx +} + func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte, shard uint64, viewName string, block int) error { span, ctx := tracing.StartSpanFromContext(ctx, "Field.importRoaringOverwrite") defer span.Finish() @@ -1787,7 +1802,7 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte, switch f.Options().Type { case FieldTypeInt, FieldTypeDecimal: frag.mu.Lock() - if err := frag.calculateMaxRowID(); err != nil { + if err := frag.calculateMaxRowID(tx); err != nil { return err } maxRowID, _, err := frag.maxRow(tx, nil) @@ -2132,6 +2147,9 @@ func isValidCacheType(v string) bool { } } +// TODO(jea): why isn't this bits.Len64(x) using import "math/bits" +// That would be much (80x or more) faster and correct if the high bit is set. +// // bitDepth returns the number of bits required to store a value. func bitDepth(v uint64) uint { for i := uint(0); i < 63; i++ { diff --git a/field_internal_test.go b/field_internal_test.go index 8a5aaffe4..0472e7dcc 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -205,10 +205,17 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField { if err != nil { t.Fatal(err) } - field, err := NewField(NewHolder(DefaultPartitionN), path, "i", "f", opts) + h := NewHolder(DefaultPartitionN) + h.Path = path + idx, err := h.CreateIndex("i", IndexOptions{}) + if err != nil { + panic(err) + } + field, err := NewField(h, path, "i", "f", opts) if err != nil { t.Fatal(err) } + field.idx = idx return &TestField{Field: field} } @@ -223,6 +230,9 @@ func OpenField(t *testing.T, opts FieldOption) *TestField { // Close closes the field and removes the underlying data. func (f *TestField) Close() error { + if f.idx != nil { + panicOn(f.idx.Txf.CloseIndex(f.idx)) + } defer os.RemoveAll(f.Path()) return f.Field.Close() } @@ -235,10 +245,17 @@ func (f *TestField) Reopen() error { } path, index, name := f.Path(), f.Index(), f.Name() - f.Field, err = NewField(NewHolder(DefaultPartitionN), path, index, name, OptFieldTypeDefault()) + h := NewHolder(DefaultPartitionN) + h.Path = path + idx, err := h.CreateIndex(index, IndexOptions{}) if err != nil { return err } + f.Field, err = NewField(h, path, index, name, OptFieldTypeDefault()) + if err != nil { + return err + } + f.Field.idx = idx if err := f.Open(); err != nil { return err @@ -311,7 +328,8 @@ func TestField_RowTime(t *testing.T) { defer f.Close() // Obtain transaction. - tx := &RoaringTx{Field: f.Field} + tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field}) + defer tx.Rollback() if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil { t.Fatal(err) @@ -323,6 +341,12 @@ func TestField_RowTime(t *testing.T) { f.MustSetBit(tx, 1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC)) f.MustSetBit(tx, 1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC)) + panicOn(tx.Commit()) + + // obtain 2nd transaction to read it back. + tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field}) + defer tx.Rollback() + if r, err := f.RowTime(tx, 1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(r.Columns(), []uint64{1, 3, 4, 5}) { @@ -358,6 +382,7 @@ func TestField_RowTime(t *testing.T) { func TestField_PersistAvailableShards(t *testing.T) { availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write f := OpenField(t, OptFieldTypeDefault()) + defer f.Close() // bm represents remote available shards. bm := roaring.NewBitmap(1, 2, 3) @@ -379,6 +404,7 @@ func TestField_PersistAvailableShards(t *testing.T) { func TestField_CorruptAvailableShards(t *testing.T) { availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write f := OpenField(t, OptFieldTypeDefault()) + defer f.Close() // bm represents remote available shards. bm := roaring.NewBitmap(1, 2, 3) @@ -411,6 +437,7 @@ func TestField_CorruptAvailableShards(t *testing.T) { func TestField_TruncatedAvailableShards(t *testing.T) { availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write f := OpenField(t, OptFieldTypeDefault()) + defer f.Close() // bm represents remote available shards. bm := roaring.NewBitmap(1, 2, 3) @@ -441,6 +468,7 @@ func TestField_TruncatedAvailableShards(t *testing.T) { func TestField_PersistAvailableShardsFootprint(t *testing.T) { availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write f := OpenField(t, OptFieldTypeDefault()) + defer f.Close() // bm represents remote available shards. bm := roaring.NewBitmap() @@ -554,6 +582,7 @@ func TestField_ApplyOptions(t *testing.T) { // to result in a value of 9 instead of 1. func TestBSIGroup_importValue(t *testing.T) { f := OpenField(t, OptFieldTypeInt(-100, 200)) + defer f.Close() options := &ImportOptions{} for i, tt := range []struct { @@ -581,22 +610,31 @@ func TestBSIGroup_importValue(t *testing.T) { []uint64{100}, }, } { - tx := &RoaringTx{Field: f.Field} + tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field}) + // can't do this, we are in a loop, not a function: + // defer tx.Rollback() if err := f.importValue(tx, tt.columnIDs, tt.values, options); err != nil { t.Fatalf("test %d, importing values: %s", i, err.Error()) } + panicOn(tx.Commit()) + + tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field}) + // no, same reason as above: defer tx.Rollback() + if row, err := f.Range(tx, f.name, pql.EQ, tt.checkVal); err != nil { t.Fatalf("test %d, getting range: %s", i, err.Error()) } else if !reflect.DeepEqual(row.Columns(), tt.expCols) { t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns()) } - } + tx.Rollback() + } // loop } func TestIntField_MinMaxForShard(t *testing.T) { f := OpenField(t, OptFieldTypeInt(-100, 200)) + defer f.Close() options := &ImportOptions{} for i, test := range []struct { @@ -648,12 +686,17 @@ func TestIntField_MinMaxForShard(t *testing.T) { }, } { t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { - tx := &RoaringTx{Field: f.Field} + tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field}) + defer tx.Rollback() if err := f.importValue(tx, test.columnIDs, test.values, options); err != nil { t.Fatalf("test %d, importing values: %s", i, err.Error()) } + panicOn(tx.Commit()) + tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field}) + defer tx.Rollback() + maxvc, err := f.MaxForShard(tx, 0, nil) if err != nil { t.Fatalf("getting max for shard: %v", err) @@ -753,6 +796,7 @@ func TestDecimalField_MinMaxBoundaries(t *testing.T) { func TestDecimalField_MinMaxForShard(t *testing.T) { f := OpenField(t, OptFieldTypeDecimal(3)) + defer f.Close() options := &ImportOptions{} for i, test := range []struct { @@ -804,12 +848,17 @@ func TestDecimalField_MinMaxForShard(t *testing.T) { }, } { t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { - tx := &RoaringTx{Field: f.Field} + tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field}) + defer tx.Rollback() if err := f.importFloatValue(tx, test.columnIDs, test.values, options); err != nil { t.Fatalf("test %d, importing values: %s", i, err.Error()) } + panicOn(tx.Commit()) + tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field}) + defer tx.Rollback() + maxvc, err := f.MaxForShard(tx, 0, nil) if err != nil { t.Fatalf("getting max for shard: %v", err) diff --git a/field_test.go b/field_test.go index aeade9fcd..2fc804b7c 100644 --- a/field_test.go +++ b/field_test.go @@ -25,6 +25,8 @@ import ( "github.com/pilosa/pilosa/v2/test" ) +var panicOn = pilosa.PanicOn + // Ensure a field can set & read a bsiGroup value. func TestField_SetValue(t *testing.T) { t.Run("OK", func(t *testing.T) { @@ -35,7 +37,10 @@ func TestField_SetValue(t *testing.T) { if err != nil { t.Fatal(err) } - tx := &pilosa.RoaringTx{Field: f.Field} + + idxPilosa := f.Field.GetIndex() + tx := idxPilosa.NewTx(pilosa.Txo{Write: writable, Index: idxPilosa, Field: f.Field}) + defer tx.Rollback() // Set value on field. if changed, err := f.SetValue(tx, 100, 21); err != nil { @@ -69,7 +74,9 @@ func TestField_SetValue(t *testing.T) { if err != nil { t.Fatal(err) } - tx := &pilosa.RoaringTx{Field: f.Field} + idxP := f.Field.GetIndex() + tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field}) + defer tx.Rollback() // Set value. if changed, err := f.SetValue(tx, 100, 21); err != nil { @@ -103,7 +110,9 @@ func TestField_SetValue(t *testing.T) { if err != nil { t.Fatal(err) } - tx := &pilosa.RoaringTx{Field: f.Field} + idxP := f.Field.GetIndex() + tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field}) + defer tx.Rollback() // Set value. if _, err := f.SetValue(tx, 100, 21); err != pilosa.ErrBSIGroupNotFound { @@ -119,7 +128,9 @@ func TestField_SetValue(t *testing.T) { if err != nil { t.Fatal(err) } - tx := &pilosa.RoaringTx{Field: f.Field} + idxP := f.Field.GetIndex() + tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field}) + defer tx.Rollback() // Set value. if _, err := f.SetValue(tx, 100, 15); err != pilosa.ErrBSIGroupValueTooLow { @@ -135,7 +146,9 @@ func TestField_SetValue(t *testing.T) { if err != nil { t.Fatal(err) } - tx := &pilosa.RoaringTx{Field: f.Field} + idxP := f.Field.GetIndex() + tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field}) + defer tx.Rollback() // Set value. if _, err := f.SetValue(tx, 100, 31); err != pilosa.ErrBSIGroupValueTooHigh { @@ -204,7 +217,9 @@ func TestField_AvailableShards(t *testing.T) { if err != nil { t.Fatal(err) } - tx := &pilosa.RoaringTx{Field: f.Field} + idxP := f.Field.GetIndex() + tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field}) + defer tx.Rollback() // Set values on shards 0 & 2, and verify. if _, err := f.SetBit(tx, 0, 100, nil); err != nil { @@ -214,6 +229,7 @@ func TestField_AvailableShards(t *testing.T) { } else if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" { t.Fatal(diff) } + panicOn(tx.Commit()) // Set remote shards and verify. if err := f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)); err != nil { @@ -244,7 +260,9 @@ func TestField_ClearValue(t *testing.T) { if err != nil { t.Fatal(err) } - tx := &pilosa.RoaringTx{Field: f.Field} + idxP := f.Field.GetIndex() + tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field}) + defer tx.Rollback() // Set value on field. if changed, err := f.SetValue(tx, 100, 21); err != nil { @@ -252,6 +270,9 @@ func TestField_ClearValue(t *testing.T) { } else if !changed { t.Fatal("expected change") } + panicOn(tx.Commit()) + + tx = idxP.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idxP, Field: f.Field}) // Read value. if value, exists, err := f.Value(tx, 100); err != nil { @@ -261,12 +282,17 @@ func TestField_ClearValue(t *testing.T) { } else if !exists { t.Fatal("expected value to exist") } + tx.Rollback() + tx = idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field}) if changed, err := f.ClearValue(tx, 100); err != nil { t.Fatal(err) } else if !changed { t.Fatal(err) } + panicOn(tx.Commit()) + tx = idxP.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idxP, Field: f.Field}) + defer tx.Rollback() // Read value. if _, exists, err := f.Value(tx, 100); err != nil { diff --git a/fragment.go b/fragment.go index 9734429ed..c5991427c 100644 --- a/fragment.go +++ b/fragment.go @@ -108,6 +108,9 @@ type fragment struct { view string shard uint64 + // idx cached to avoid repeatedly looking it up everywhere. + idx *Index + // parent holder, used to find snapshot queue, etc. holder *Holder @@ -180,6 +183,10 @@ type fragment struct { // newFragment returns a new instance of Fragment. func newFragment(holder *Holder, path, index, field, view string, shard uint64, flags byte) *fragment { + idx := holder.Index(index) + if idx == nil { + panic(fmt.Sprintf("holder=%#v but got nil idx back from holder!", holder)) + } f := &fragment{ path: path, index: index, @@ -187,6 +194,7 @@ func newFragment(holder *Holder, path, index, field, view string, shard uint64, view: view, shard: shard, flags: flags, + idx: idx, CacheType: DefaultCacheType, CacheSize: DefaultCacheSize, @@ -250,7 +258,9 @@ func (f *fragment) Open() error { f.checksums = make(map[int][]byte) // Read last bit to determine max row. - return f.calculateMaxRowID() + tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f}) + defer tx.Rollback() + return f.calculateMaxRowID(tx) }(); err != nil { f.close() return err @@ -398,6 +408,15 @@ func (f *fragment) inspectStorage(data []byte, file *os.File, newGen generation, // logic is now mostly in importStorage (reading in a bitmap) and applyStorage // (remapping an existing bitmap to match a new backing store). func (f *fragment) openStorage(unmarshalData bool) error { + + if !f.idx.NeedsSnapshot() { + f.gen = &NopGeneration{} + f.rowCache = &simpleCache{make(map[uint64]*Row)} + f.currdata = struct{ from, to uintptr }{} + f.prevdata = f.currdata + return nil // openStorage becomes a noop under RBF, Badger, etc. + } + // Create a roaring bitmap to serve as storage for the shard. if f.storage == nil { f.storage = roaring.NewFileBitmap() @@ -473,10 +492,16 @@ func (f *fragment) openCache() error { return nil } + tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f}) + defer tx.Rollback() + // Read in all rows by ID. // This will cause them to be added to the cache. for _, id := range pb.IDs { - n := f.storage.CountRange(id*ShardWidth, (id+1)*ShardWidth) + n, err := tx.CountRange(f.index, f.field, f.view, f.shard, id*ShardWidth, (id+1)*ShardWidth) + if err != nil { + return errors.Wrap(err, "CountRange") + } f.cache.BulkAdd(id, n) } f.cache.Invalidate() @@ -583,7 +608,7 @@ func (f *fragment) rowFromStorage(tx Tx, rowID uint64) (*Row, error) { row := &Row{ segments: []rowSegment{{ - data: data, // this data contains BadgerTx data, which should not survive Txn commit. + data: data, shard: f.shard, writable: true, }}, @@ -598,7 +623,11 @@ func (f *fragment) rowFromStorage(tx Tx, rowID uint64) (*Row, error) { func (f *fragment) setBit(tx Tx, rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - err = f.gen.Transaction(&f.storage.OpWriter, func() error { + var wp *io.Writer + if f.storage != nil { + wp = &f.storage.OpWriter + } + err = f.gen.Transaction(wp, func() error { // handle mutux field type if f.mutexVector != nil { if err := f.handleMutex(tx, rowID, columnID); err != nil { @@ -680,7 +709,11 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo func (f *fragment) clearBit(tx Tx, rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - err = f.gen.Transaction(&f.storage.OpWriter, func() error { + var wp *io.Writer + if f.storage != nil { + wp = &f.storage.OpWriter + } + err = f.gen.Transaction(wp, func() error { changed, err = f.unprotectedClearBit(tx, rowID, columnID) return err }) @@ -739,7 +772,11 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b func (f *fragment) setRow(tx Tx, row *Row, rowID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - err = f.gen.Transaction(&f.storage.OpWriter, func() error { + var wp *io.Writer + if f.storage != nil { + wp = &f.storage.OpWriter + } + err = f.gen.Transaction(wp, func() error { changed, err = f.unprotectedSetRow(tx, row, rowID) return err }) @@ -802,7 +839,11 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo func (f *fragment) clearRow(tx Tx, rowID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - err = f.gen.Transaction(&f.storage.OpWriter, func() error { + var wp *io.Writer + if f.storage != nil { + wp = &f.storage.OpWriter + } + err = f.gen.Transaction(wp, func() error { changed, err = f.unprotectedClearRow(tx, rowID) return err }) @@ -845,7 +886,11 @@ func (f *fragment) unprotectedClearRow(tx Tx, rowID uint64) (changed bool, err e // This updates both the on-disk storage and the in-cache bitmap. func (f *fragment) unprotectedClearBlock(tx Tx, block int) (changed bool, err error) { firstRow := uint64(block * HashBlockSize) - err = f.gen.Transaction(&f.storage.OpWriter, func() error { + var wp *io.Writer + if f.storage != nil { + wp = &f.storage.OpWriter + } + err = f.gen.Transaction(wp, func() error { var rowChanged bool for rowID := uint64(firstRow); rowID < firstRow+HashBlockSize; rowID++ { if changed, err := f.unprotectedClearRow(tx, rowID); err != nil { @@ -953,8 +998,11 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64 func (f *fragment) setValueBase(tx Tx, columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - - err = f.gen.Transaction(&f.storage.OpWriter, func() error { + var wp *io.Writer + if f.storage != nil { + wp = &f.storage.OpWriter + } + err = f.gen.Transaction(wp, func() error { // Convert value to an unsigned representation. uvalue := uint64(value) if value < 0 { @@ -1291,8 +1339,12 @@ func (f *fragment) maxRow(tx Tx, filter *Row) (uint64, uint64, error) { // calculateMaxRowID determines the field's maxRowID value based // on the contents of its storage, and sets the struct argument. -func (f *fragment) calculateMaxRowID() (err error) { - f.maxRowID = f.storage.Max() / ShardWidth +func (f *fragment) calculateMaxRowID(tx Tx) (err error) { + max, err := tx.Max(f.index, f.field, f.view, f.shard) + if err != nil { + return err + } + f.maxRowID = max / ShardWidth return nil } @@ -1484,7 +1536,6 @@ func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality if err != nil { return nil, err } - // Create predicate without sign bit. upredicate := absInt64(predicate) @@ -1492,7 +1543,6 @@ func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality if err != nil { return nil, err } - switch { case predicate == 0 && !allowEquality: // Match all positive numbers except zero. @@ -2217,7 +2267,11 @@ func (f *fragment) bulkImportStandard(tx Tx, rowIDs, columnIDs []uint64, options // operations to the op log. func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64]struct{}) error { //tx.AddN() - err := f.gen.Transaction(&f.storage.OpWriter, func() error { + var wp *io.Writer + if f.storage != nil { + wp = &f.storage.OpWriter + } + err := f.gen.Transaction(wp, func() error { // segfault if len(set) > 0 { f.stats.Count(MetricImportingN, int64(len(set)), 1) @@ -2453,14 +2507,18 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b span, ctx := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits") var changed int var rowSet map[uint64]int - err := f.gen.Transaction(&f.storage.OpWriter, func() (err error) { + var wp *io.Writer + if f.storage != nil { + wp = &f.storage.OpWriter + } + err := f.gen.Transaction(wp, func() (err error) { var rit roaring.RoaringIterator rit, err = roaring.NewRoaringIterator(data) if err != nil { return err } - changed, rowSet, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, rit, clear, true, rowSize) + changed, rowSet, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, rit, clear, true, rowSize, nil) return err }) @@ -2550,6 +2608,9 @@ func track(start time.Time, message string, stats stats.StatsClient, logger logg // snapshot does the actual snapshot operation. it does not check or care // about f.snapshotPending. func (f *fragment) snapshot() (err error) { + if !f.idx.NeedsSnapshot() { + return nil + } if !f.open { return errors.New("snapshot request on closed fragment") } @@ -2688,31 +2749,16 @@ func (f *fragment) WriteTo(w io.Writer) (n int64, err error) { return 0, nil } +// used in shipping the slices across the network for a resize. func (f *fragment) writeStorageToArchive(tw *tar.Writer) error { - // Open separate file descriptor to read from. - file, err := os.Open(f.path) + + tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx}) + defer tx.Rollback() + file, sz, err := tx.RoaringBitmapReader(f.index, f.field, f.view, f.shard, f.path) if err != nil { - return errors.Wrap(err, "opening file") - } - defer file.Close() - - // Retrieve the current file size under lock so we don't read - // while an operation is appending to the end. - var sz int64 - if err := func() error { - f.mu.Lock() - defer f.mu.Unlock() - - fi, err := file.Stat() - if err != nil { - return errors.Wrap(err, "statting") - } - sz = fi.Size() - - return nil - }(); err != nil { return err } + defer file.Close() // Write archive header. if err := tw.WriteHeader(&tar.Header{ @@ -2779,9 +2825,15 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) { // Process file based on file name. switch hdr.Name { case "data": - if err := f.readStorageFromArchive(tr); err != nil { + idx := f.holder.Index(f.index) + tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f}) + defer tx.Rollback() + if err := f.fillFragmentFromArchive(tx, tr); err != nil { return 0, errors.Wrap(err, "reading storage") } + if err := tx.Commit(); err != nil { + return 0, errors.Wrap(err, "Commit after tx.ReadFragmentFromArchive") + } case "cache": if err := f.readCacheFromArchive(tr); err != nil { return 0, errors.Wrap(err, "reading cache") @@ -2794,7 +2846,40 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) { return 0, nil } +// should be morally equivalent to fragment.readStorageFromArchive() +// below for RoaringTx, but also work on any Tx because it uses +// tx.ImportRoaringBits(). +func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error { + + // this is reading from inside a tarball, so definitely no need + // to close it here. + data, err := ioutil.ReadAll(r) + if err != nil { + return errors.Wrap(err, "fillFragmentFromArchive ioutil.ReadAll(r)") + } + if len(data) == 0 { + return nil + } + + // For reference, compare to what fragment.go:313 fragment.importStorage() does. + + clear := false + log := false + rowSize := uint64(0) + itr, err := roaring.NewRoaringIterator(data) + if err != nil { + return errors.Wrap(err, "fillFragmentFromArchive NewRoaringIterator") + } + changed, rowSet, err := tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, itr, clear, log, rowSize, data) + _, _ = changed, rowSet + if err != nil { + return errors.Wrap(err, "fillFragmentFromArchive ImportRoaringBits") + } + return nil +} + func (f *fragment) readStorageFromArchive(r io.Reader) error { + // Create a temporary file to copy into. path := f.path + copyExt file, err := os.Create(path) @@ -2808,6 +2893,12 @@ func (f *fragment) readStorageFromArchive(r io.Reader) error { return errors.Wrap(err, "copying") } + // TODO(jea): isn't this next Rename a file handle leak? + // try closing first + if err := f.closeStorage(); err != nil { + return errors.Wrap(err, "closeStorage-prior-to-Rename-and-openStorage") + } + // Move snapshot to data file location. if err := os.Rename(path, f.path); err != nil { return errors.Wrap(err, "renaming") @@ -2934,13 +3025,15 @@ func (f *fragment) rows(ctx context.Context, tx Tx, start uint64, filters ...row // unprotectedRows calls rows without grabbing the mutex. func (f *fragment) unprotectedRows(ctx context.Context, tx Tx, start uint64, filters ...rowFilter) ([]uint64, error) { + rows := make([]uint64, 0) startKey := rowToKey(start) i, _, err := tx.ContainerIterator(f.index, f.field, f.view, f.shard, startKey) if err != nil { return nil, err + } else if i == nil { + return rows, nil } defer i.Close() // must close iterators allocated on a Tx - rows := make([]uint64, 0) var lastRow uint64 = math.MaxUint64 // Loop over the existing containers. diff --git a/fragment_internal_test.go b/fragment_internal_test.go index dca084b5a..871864d3d 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -24,6 +24,7 @@ import ( "math" "math/rand" "os" + "path/filepath" "reflect" "runtime" "runtime/debug" @@ -445,7 +446,7 @@ func TestFragment_SetValue(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set value. if changed, err := f.setValue(tx, 100, 10, 20); err != nil { @@ -481,7 +482,7 @@ func TestFragment_SetValue(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set values. m := make(map[uint64]int64) @@ -538,12 +539,11 @@ func TestFragment_Sum(t *testing.T) { const bitDepth = 16 f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") - _ = idx defer f.Clean(t) // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set values. vals := []struct { @@ -562,6 +562,10 @@ func TestFragment_Sum(t *testing.T) { } } + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + t.Run("NoFilter", func(t *testing.T) { if sum, n, err := f.sum(tx, nil, bitDepth); err != nil { t.Fatal(err) @@ -582,10 +586,19 @@ func TestFragment_Sum(t *testing.T) { } }) + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f}) + defer tx.Rollback() + // verify that clearValue clears values if _, err := f.clearValue(tx, 1000, bitDepth, 23); err != nil { t.Fatal(err) } + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + t.Run("ClearValue", func(t *testing.T) { if sum, n, err := f.sum(tx, nil, bitDepth); err != nil { t.Fatal(err) @@ -602,12 +615,11 @@ func TestFragment_MinMax(t *testing.T) { const bitDepth = 16 f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") - _ = idx defer f.Clean(t) // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set values. if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { @@ -626,6 +638,12 @@ func TestFragment_MinMax(t *testing.T) { t.Fatal(err) } + panicOn(tx.Commit()) + + // the new tx is shared by Min/Max below. + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + t.Run("Min", func(t *testing.T) { tests := []struct { filter *Row @@ -689,7 +707,7 @@ func TestFragment_Range(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set values. if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { @@ -716,7 +734,8 @@ func TestFragment_Range(t *testing.T) { defer f.Clean(t) // Obtain transaction. - tx := &RoaringTx{fragment: f} + tx := f.txTestingOnly + defer tx.Rollback() // Set values. if _, err := f.setValue(tx, 1000, 1, 0); err != nil { @@ -745,7 +764,7 @@ func TestFragment_Range(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set values. if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { @@ -773,7 +792,7 @@ func TestFragment_Range(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set values. if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { @@ -826,7 +845,7 @@ func TestFragment_Range(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() if _, err := f.setValue(tx, 1, 1, 1); err != nil { t.Fatal(err) @@ -846,7 +865,7 @@ func TestFragment_Range(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() if _, err := f.setValue(tx, 1, 2, 3); err != nil { t.Fatal(err) @@ -868,7 +887,7 @@ func TestFragment_Range(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set values. if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { @@ -921,7 +940,7 @@ func TestFragment_Range(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() if _, err := f.setValue(tx, 1, 2, 0); err != nil { t.Fatal(err) @@ -942,7 +961,8 @@ func TestFragment_Range(t *testing.T) { defer f.Clean(t) // Obtain transaction. - tx := &RoaringTx{fragment: f} + tx := f.txTestingOnly + defer tx.Rollback() if _, err := f.setValue(tx, 1, 2, 0); err != nil { t.Fatal(err) @@ -964,7 +984,7 @@ func TestFragment_Range(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set values. if _, err := f.setValue(tx, 1000, bitDepth, 382); err != nil { @@ -1016,7 +1036,8 @@ func TestFragment_Range(t *testing.T) { defer f.Clean(t) // Obtain transaction. - tx := &RoaringTx{fragment: f} + tx := f.txTestingOnly + defer tx.Rollback() if _, err := f.setValue(tx, 1, 64, 0xf0); err != nil { t.Fatal(err) @@ -1037,7 +1058,7 @@ func TestFragment_Range(t *testing.T) { func benchmarkSetValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() column := uint64(0) for i := 0; i < b.N; i++ { @@ -1074,7 +1095,7 @@ func BenchmarkFragment_SetValue(b *testing.B) { func benchmarkImportValues(b *testing.B, bitDepth uint, f *fragment, cfunc func(uint64) uint64) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() column := uint64(0) b.StopTimer() @@ -1145,7 +1166,7 @@ func BenchmarkFragment_RepeatedSmallImports(b *testing.B) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() err := f.importRoaringT(tx, getZipfRowsSliceRoaring(uint64(numRows), 1, 0, ShardWidth), false) if err != nil { @@ -1162,6 +1183,7 @@ func BenchmarkFragment_RepeatedSmallImports(b *testing.B) { b.Fatalf("doing small bulk import: %v", err) } } + tx.Rollback() // don't exhaust the Tx space under b.N iterations. } }) } @@ -1239,7 +1261,7 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() err := f.importValue(tx, initialCols, initialVals, 21, false) if err != nil { @@ -1257,6 +1279,7 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) { b.Fatalf("importing values: %v", err) } } + tx.Rollback() // don't exhaust the Tx over the b.N iterations. } }) } @@ -1268,12 +1291,11 @@ func BenchmarkFragment_RepeatedSmallValueImports(b *testing.B) { // Ensure a fragment can snapshot correctly. func TestFragment_Snapshot(t *testing.T) { f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") - _ = idx defer f.Clean(t) // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set and then clear bits on the fragment. if _, err := f.setBit(tx, 1000, 1); err != nil { @@ -1283,6 +1305,9 @@ func TestFragment_Snapshot(t *testing.T) { } else if _, err := f.clearBit(tx, 1000, 1); err != nil { t.Fatal(err) } + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { @@ -1307,7 +1332,7 @@ func TestFragment_ForEachBit(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set bits on the fragment. if _, err := f.setBit(tx, 100, 20); err != nil { @@ -1341,7 +1366,7 @@ func TestFragment_Top(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set bits on the rows 100, 101, & 102. f.mustSetBits(tx, 100, 1, 3, 200) @@ -1364,12 +1389,11 @@ func TestFragment_Top(t *testing.T) { // Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { f, idx := mustOpenFragment("i", "f", viewStandard, 0, CacheTypeRanked) - _ = idx defer f.Clean(t) // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set bits on the rows 100, 101, & 102. f.mustSetBits(tx, 100, 1, 3, 200) @@ -1386,6 +1410,10 @@ func TestFragment_Top_Filter(t *testing.T) { t.Fatalf("setAttrs: %v", err) } + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + // Retrieve top rows. if pairs, err := f.top(tx, topOptions{ N: 2, @@ -1410,7 +1438,7 @@ func TestFragment_TopN_Intersect(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Create an intersecting input row. src := NewRow(1, 2, 3) @@ -1446,7 +1474,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Create an intersecting input row. src := NewRow( @@ -1499,7 +1527,7 @@ func TestFragment_TopN_IDs(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set bits on various rows. f.mustSetBits(tx, 100, 1, 2, 3) @@ -1525,7 +1553,7 @@ func TestFragment_TopN_NopCache(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set bits on various rows. f.mustSetBits(tx, 100, 1, 2, 3) @@ -1700,7 +1728,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { defer f.Clean(t) // Obtain transaction. - tx := f.txTestingOnly //&RoaringTx{fragment: f} + tx := f.txTestingOnly defer tx.Rollback() // Set bits on a different block. @@ -1727,7 +1755,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { @@ -1743,6 +1771,8 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { t.Fatalf("unexpected cache len: %d", cache.Len()) } + panicOn(tx.Commit()) + // Reopen the fragment. if err := f.Reopen(); err != nil { t.Fatal(err) @@ -1758,6 +1788,8 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Ensure a fragment's cache can be persisted between restarts. func TestFragment_RankCache_Persistence(t *testing.T) { + skipForRBF(t) + index := mustOpenIndex(IndexOptions{}) defer index.Close() @@ -1781,6 +1813,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Obtain transaction. tx := index.Txf.NewTx(Txo{Write: writable, Index: index, Fragment: f}) + defer tx.Rollback() // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { @@ -1789,6 +1822,10 @@ func TestFragment_RankCache_Persistence(t *testing.T) { } } + panicOn(tx.Commit()) + tx = index.Txf.NewTx(Txo{Write: !writable, Index: index, Fragment: f}) + defer tx.Rollback() + // Verify correct cache type and size. if cache, ok := f.cache.(*rankCache); !ok { t.Fatalf("unexpected cache: %T", f.cache) @@ -1812,14 +1849,28 @@ func TestFragment_RankCache_Persistence(t *testing.T) { } } +func roaringOnlyTest(t *testing.T) { + if os.Getenv("PILOSA_TXSRC") != "roaring" { + t.Skip("skip for everything but roaring") + } +} + +func roaringOnlyBenchmark(b *testing.B) { + if os.Getenv("PILOSA_TXSRC") != "roaring" { + b.Skip("skip for everything but roaring") + } +} + // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { + roaringOnlyTest(t) + f0, idx := mustOpenFragment("i", "f", viewStandard, 0, "") _ = idx defer f0.Clean(t) - // Obtain transaction. - tx := &RoaringTx{fragment: f0} + tx := f0.txTestingOnly + defer tx.Rollback() // Set and then clear bits on the fragment. if _, err := f0.setBit(tx, 1000, 1); err != nil { @@ -1847,7 +1898,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { _ = idx defer f1.Clean(t) - if rn, err := f1.ReadFrom(&buf); err != nil { + if rn, err := f1.ReadFrom(&buf); err != nil { // eventually calls fragment.fillFragmentFromArchive t.Fatal(err) } else if wn != rn { t.Fatalf("read/write byte count mismatch: wn=%d, rn=%d", wn, rn) @@ -1898,13 +1949,12 @@ func BenchmarkFragment_Blocks(b *testing.B) { func BenchmarkFragment_IntersectionCount(b *testing.B) { f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") - _ = idx defer f.Clean(b) f.MaxOpN = math.MaxInt32 // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Generate some intersecting data. for i := 0; i < 10000; i += 2 { @@ -1918,6 +1968,10 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { } } + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + // Snapshot to disk before benchmarking. if err := f.Snapshot(); err != nil { b.Fatal(err) @@ -1939,7 +1993,7 @@ func TestFragment_Tanimoto(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() src := NewRow(1, 2, 3) @@ -1967,7 +2021,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() src := NewRow(1, 2, 3) @@ -1991,13 +2045,15 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { } func TestFragment_Snapshot_Run(t *testing.T) { + roaringOnlyTest(t) + f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set bits on the fragment. for i := uint64(1); i < 3; i++ { @@ -2029,7 +2085,7 @@ func TestFragment_SetMutex(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() var cols []uint64 @@ -2148,7 +2204,7 @@ func TestFragment_ImportSet(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set import. err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -2181,6 +2237,140 @@ func TestFragment_ImportSet(t *testing.T) { } } +func TestFragment_ImportSet_WithTxCommit(t *testing.T) { + tests := []struct { + setRowIDs []uint64 + setColIDs []uint64 + setExp map[uint64][]uint64 + clearRowIDs []uint64 + clearColIDs []uint64 + clearExp map[uint64][]uint64 + }{ + { + []uint64{1, 1, 1, 1}, + []uint64{0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + }, + []uint64{}, + []uint64{}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{1, 1, 1, 1, 2, 2, 2, 2}, + []uint64{0, 1, 2, 3, 0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + 2: {0, 1, 2, 3}, + }, + []uint64{1, 1, 2}, + []uint64{1, 2, 3}, + map[uint64][]uint64{ + 1: {0, 3}, + 2: {0, 1, 2}, + }, + }, + { + []uint64{1, 1, 1, 1, 2}, + []uint64{0, 1, 2, 3, 1}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + 2: {1}, + }, + []uint64{1, 1, 1, 1}, + []uint64{0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {}, + 2: {1}, + }, + }, + { + []uint64{1, 1, 1, 1, 2, 2, 1}, + []uint64{0, 1, 2, 3, 1, 8, 1}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + 2: {1, 8}, + }, + []uint64{1, 1}, + []uint64{0, 0}, + map[uint64][]uint64{ + 1: {1, 2, 3}, + 2: {1, 8}, + }, + }, + { + []uint64{1, 2, 3}, + []uint64{8, 8, 8}, + map[uint64][]uint64{ + 1: {8}, + 2: {8}, + 3: {8}, + }, + []uint64{1, 2, 3}, + []uint64{9, 9, 9}, + map[uint64][]uint64{ + 1: {8}, + 2: {8}, + 3: {8}, + }, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("importset%d", i), func(t *testing.T) { + f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") + _ = idx + defer f.Clean(t) + + // Obtain transaction. + tx := f.txTestingOnly + defer tx.Rollback() + + // Set import. + err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) + if err != nil { + t.Fatalf("bulk importing ids: %v", err) + } + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + // Check for expected results. + for k, v := range test.setExp { + cols := f.mustRow(tx, k).Columns() + if !reflect.DeepEqual(cols, v) { + t.Fatalf("expected: %v, but got: %v", v, cols) + } + } + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + // Clear import. + err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) + if err != nil { + t.Fatalf("bulk clearing ids: %v", err) + } + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + // Check for expected results. + for k, v := range test.clearExp { + cols := f.mustRow(tx, k).Columns() + if !reflect.DeepEqual(cols, v) { + t.Fatalf("expected: %v, but got: %v", v, cols) + } + } + }) + } +} + func TestFragment_ConcurrentImport(t *testing.T) { t.Run("bulkImportStandard", func(t *testing.T) { f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") @@ -2189,7 +2379,7 @@ func TestFragment_ConcurrentImport(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() eg := errgroup.Group{} eg.Go(func() error { return f.bulkImportStandard(tx, []uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) }) @@ -2291,7 +2481,7 @@ func TestFragment_ImportMutex(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set import. err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -2324,6 +2514,141 @@ func TestFragment_ImportMutex(t *testing.T) { } } +// Ensure a fragment can import mutually exclusive values. +// Now with Commits in the middle. +func TestFragment_ImportMutex_WithTxCommit(t *testing.T) { + tests := []struct { + setRowIDs []uint64 + setColIDs []uint64 + setExp map[uint64][]uint64 + clearRowIDs []uint64 + clearColIDs []uint64 + clearExp map[uint64][]uint64 + }{ + { + []uint64{1, 1, 1, 1}, + []uint64{0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + }, + []uint64{}, + []uint64{}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{1, 1, 1, 1, 2, 2, 2, 2}, + []uint64{0, 1, 2, 3, 0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {}, + 2: {0, 1, 2, 3}, + }, + []uint64{1, 1, 2}, + []uint64{1, 2, 3}, + map[uint64][]uint64{ + 1: {}, + 2: {0, 1, 2}, + }, + }, + { + []uint64{1, 1, 1, 1, 2}, + []uint64{0, 1, 2, 3, 1}, + map[uint64][]uint64{ + 1: {0, 2, 3}, + 2: {1}, + }, + []uint64{1, 1, 1, 1}, + []uint64{0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {}, + 2: {1}, + }, + }, + { + []uint64{1, 1, 1, 1, 2, 2, 1}, + []uint64{0, 1, 2, 3, 1, 8, 1}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + 2: {8}, + }, + []uint64{1, 1}, + []uint64{0, 0}, + map[uint64][]uint64{ + 1: {1, 2, 3}, + 2: {8}, + }, + }, + { + []uint64{1, 2, 3}, + []uint64{8, 8, 8}, + map[uint64][]uint64{ + 1: {}, + 2: {}, + 3: {8}, + }, + []uint64{1, 2, 3}, + []uint64{9, 9, 9}, + map[uint64][]uint64{ + 1: {}, + 2: {}, + 3: {8}, + }, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { + f, idx := mustOpenMutexFragment("i", "f", viewStandard, 0, "") + defer f.Clean(t) + + // Obtain transaction. + tx := f.txTestingOnly + defer tx.Rollback() + + // Set import. + err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) + if err != nil { + t.Fatalf("bulk importing ids: %v", err) + } + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + // Check for expected results. + for k, v := range test.setExp { + cols := f.mustRow(tx, k).Columns() + if !reflect.DeepEqual(cols, v) { + t.Fatalf("row: %d, expected: %v, but got: %v", k, v, cols) + } + } + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + // Clear import. + err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) + if err != nil { + t.Fatalf("bulk clearing ids: %v", err) + } + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + // Check for expected results. + for k, v := range test.clearExp { + cols := f.mustRow(tx, k).Columns() + if !reflect.DeepEqual(cols, v) { + t.Fatalf("row: %d expected: %v, but got: %v", k, v, cols) + } + } + }) + } +} + // Ensure a fragment can import bool values. func TestFragment_ImportBool(t *testing.T) { tests := []struct { @@ -2415,7 +2740,7 @@ func TestFragment_ImportBool(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // Set import. err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) @@ -2448,6 +2773,142 @@ func TestFragment_ImportBool(t *testing.T) { } } +// Ensure a fragment can import bool values, with Commits in between writes and reads. +func TestFragment_ImportBool_WithTxCommit(t *testing.T) { + tests := []struct { + setRowIDs []uint64 + setColIDs []uint64 + setExp map[uint64][]uint64 + clearRowIDs []uint64 + clearColIDs []uint64 + clearExp map[uint64][]uint64 + }{ + { + []uint64{1, 1, 1, 1}, + []uint64{0, 1, 2, 3}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + }, + []uint64{}, + []uint64{}, + map[uint64][]uint64{ + 1: {0, 1, 2, 3}, + }, + }, + { + []uint64{0, 0, 0, 0, 1, 1, 1, 1}, + []uint64{0, 1, 2, 3, 0, 1, 2, 3}, + map[uint64][]uint64{ + 0: {}, + 1: {0, 1, 2, 3}, + }, + []uint64{1, 1, 2}, + []uint64{1, 2, 3}, + map[uint64][]uint64{ + 0: {}, + 1: {0, 3}, + 2: {}, + }, + }, + { + []uint64{0, 0, 0, 0, 1}, + []uint64{0, 1, 2, 3, 1}, + map[uint64][]uint64{ + 0: {0, 2, 3}, + 1: {1}, + }, + []uint64{1, 1, 1, 1}, + []uint64{0, 1, 2, 3}, + map[uint64][]uint64{ + 0: {0, 2, 3}, + 1: {}, + }, + }, + { + []uint64{1, 1, 1, 1, 0, 0, 1}, + []uint64{0, 1, 2, 3, 1, 8, 1}, + map[uint64][]uint64{ + 0: {8}, + 1: {0, 1, 2, 3}, + }, + []uint64{1, 1}, + []uint64{0, 0}, + map[uint64][]uint64{ + 0: {8}, + 1: {1, 2, 3}, + }, + }, + { + []uint64{0, 1, 2}, + []uint64{8, 8, 8}, + map[uint64][]uint64{ + 0: {}, + 1: {}, // This isn't {8} because fragment doesn't validate bool values. + 2: {8}, + }, + []uint64{1, 2, 3}, + []uint64{9, 9, 9}, + map[uint64][]uint64{ + 0: {}, + 1: {}, + 2: {8}, + }, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("importmutex%d", i), func(t *testing.T) { + f, idx := mustOpenBoolFragment("i", "f", viewStandard, 0, "") + _ = idx + defer f.Clean(t) + + // Obtain transaction. + tx := f.txTestingOnly + defer tx.Rollback() + + // Set import. + err := f.bulkImport(tx, test.setRowIDs, test.setColIDs, &ImportOptions{}) + if err != nil { + t.Fatalf("bulk importing ids: %v", err) + } + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + // Check for expected results. + for k, v := range test.setExp { + cols := f.mustRow(tx, k).Columns() + if !reflect.DeepEqual(cols, v) { + t.Fatalf("expected: %v, but got: %v", v, cols) + } + } + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + // Clear import. + err = f.bulkImport(tx, test.clearRowIDs, test.clearColIDs, &ImportOptions{Clear: true}) + if err != nil { + t.Fatalf("bulk importing ids: %v", err) + } + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + // Check for expected results. + for k, v := range test.clearExp { + cols := f.mustRow(tx, k).Columns() + if !reflect.DeepEqual(cols, v) { + t.Fatalf("expected: %v, but got: %v", v, cols) + } + } + }) + } +} + func BenchmarkFragment_Snapshot(b *testing.B) { if *FragmentPath == "" { b.Skip("no fragment specified") @@ -2478,10 +2939,6 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { _ = idx defer f.Clean(b) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME - // Generate some intersecting data. maxX := ShardWidth / 2 sz := maxX @@ -2499,9 +2956,14 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { val += 2 i++ } + + tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + if err := f.bulkImport(tx, rows, cols, options); err != nil { b.Fatalf("Error Building Sample: %s", err) } + panicOn(tx.Commit()) if row > max { max = row } @@ -2544,14 +3006,16 @@ func BenchmarkFragment_Import(b *testing.B) { copy(rowsUse, rows) copy(colsUse, cols) f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") - _ = idx // Obtain transaction. + _ = idx + // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() b.StartTimer() if err := f.bulkImport(tx, rowsUse, colsUse, options); err != nil { b.Errorf("Error Building Sample: %s", err) } b.StopTimer() + tx.Rollback() f.Clean(b) } } @@ -2645,6 +3109,7 @@ func BenchmarkImportRoaringConcurrent(b *testing.B) { } } func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { + roaringOnlyBenchmark(b) if testing.Short() { b.SkipNow() } @@ -2664,6 +3129,7 @@ func BenchmarkImportRoaringUpdateConcurrent(b *testing.B) { // the cost of actually doing the op log for the large initial data set // is excessive. force storage into snapshotted state, then use import // to generate an op log and/or snapshot. + // note: skipped for badger, above. _, _, err := frags[j].storage.ImportRoaringBits(data, false, false, 0) if err != nil { b.Fatalf("importing roaring: %v", err) @@ -2719,7 +3185,7 @@ func BenchmarkImportStandard(b *testing.B) { _ = idx // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() b.StartTimer() err := f.bulkImport(tx, rowIDs, columnIDs, &ImportOptions{}) @@ -2727,6 +3193,7 @@ func BenchmarkImportStandard(b *testing.B) { b.Errorf("import error: %v", err) } b.StopTimer() + tx.Rollback() f.Clean(b) } }) @@ -2757,7 +3224,7 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { // to generate an op log and/or snapshot. itr, err := roaring.NewRoaringIterator(data) panicOn(err) - _, _, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, itr, false, false, 0) + _, _, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, itr, false, false, 0, nil) if err != nil { b.Errorf("import error: %v", err) } @@ -2877,14 +3344,21 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { } origF.Close() fi.Close() - nf := newFragment(NewHolder(DefaultPartitionN), fi.Name(), "i", "f", viewStandard, 0, 0) + + h := NewHolder(DefaultPartitionN) + h.Path = fi.Name() + idx, err := h.CreateIndex("i", IndexOptions{}) + panicOn(err) + + nf := newFragment(h, fi.Name(), "i", "f", viewStandard, 0, 0) err = nf.Open() if err != nil { b.Fatalf("opening fragment: %v", err) } // Obtain transaction. - tx := &RoaringTx{fragment: nf} + tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: nf}) + defer tx.Rollback() copy(rows, rowsOrig) copy(cols, colsOrig) @@ -2894,7 +3368,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { if err != nil { b.Fatalf("bulkImport: %v", err) } - + panicOn(tx.Commit()) nf.Clean(b) } } @@ -2923,13 +3397,15 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { //nf, idx := mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte) th := newTestHolder() - idx := fragTestMustOpenIndex("i", th, IndexOptions{}) + idx := fragTestMustOpenIndex(filepath.Dir(fi.Name()), "i", th, IndexOptions{}) + if th.NeedsSnapshot() { + th.SnapshotQueue = newSnapshotQueue(1, 1, nil) + } nf := newFragment(th, fi.Name(), "i", "f", viewStandard, 0, 0) tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: nf}) defer tx.Rollback() - //nf := newFragment(NewHolder(DefaultPartitionN), fi.Name(), "i", "f", viewStandard, 0, 0) err = nf.Open() if err != nil { b.Fatalf("opening fragment: %v", err) @@ -3094,6 +3570,7 @@ func BenchmarkFileWrite(b *testing.B) { ///////////////////////////////////////////////////////////////////// +// not called under badger b/c f.idx.NeedsSnapshot() in Clean() avoids it. func (f *fragment) sanityCheck(t testing.TB) { newBM := roaring.NewFileBitmap() file, err := os.Open(f.path) @@ -3110,6 +3587,7 @@ func (f *fragment) sanityCheck(t testing.TB) { t.Fatalf("sanityCheck couldn't unmarshal fragment %s: %v", f.path, err) } // Refactor fragment.storage + // note: not called for badger, see above. if equal, reason := newBM.BitwiseEqual(f.storage); !equal { t.Fatalf("fragment %s: unmarshalled bitmap different: %v", f.path, reason) } @@ -3122,25 +3600,35 @@ func (f *fragment) Clean(t testing.TB) { // check or else, in some cases, the background snapshot queue // can decide to pick it up. func() { + // should we skip snapshot queue stuff under badger/rbf? defer f.mu.Unlock() - err := defaultSnapshotQueue.Await(f) - if err != nil { - t.Fatalf("snapshot failed before sanity check: %v", err) - } - f.sanityCheck(t) - if f.storage != nil && f.storage.Source != nil { - if f.storage.Source.Dead() { - t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path, f.storage.Source.ID()) + + // badger doesn't need snapshot, so this stuff is skipped. + // The snapshot queue stuff doesn't work under badger. + if f.idx.NeedsSnapshot() { + err := defaultSnapshotQueue.Await(f) + if err != nil { + t.Fatalf("snapshot failed before sanity check: %v", err) + } + f.sanityCheck(t) + if f.storage != nil && f.storage.Source != nil { + if f.storage.Source.Dead() { + t.Fatalf("cleaning up fragment %s, source %s, source already dead", f.path, f.storage.Source.ID()) + } } } }() if f.txTestingOnly != nil { f.txTestingOnly.Rollback() + panicOn(f.idx.Txf.CloseIndex(f.idx)) } errc := f.Close() // prevent double-closes of generation during testing. f.gen = nil - errf := os.Remove(f.path) + var errf error + if FileExists(f.path) { + errf = os.Remove(f.path) // remove /var/folders/2x/hm9gp5ys3k9gmm5f_vzm_6wc0000gn/T/pilosa-index-768377904/i/f/views/standard/fragments/0: no such file or directory + } errp := os.Remove(f.cachePath()) if errc != nil || errf != nil { t.Fatal("cleaning up fragment: ", errc, errf, errp) @@ -3180,32 +3668,24 @@ func mustOpenBSIFragment(index, field, view string, shard uint64) (*fragment, *I return mustOpenFragmentFlags(index, field, view, shard, "", 1) } -var testHolder = NewHolder(DefaultPartitionN) - -func init() { - testHolder.SnapshotQueue = newSnapshotQueue(1, 1, nil) -} - func newTestHolder() *Holder { h := NewHolder(DefaultPartitionN) - h.SnapshotQueue = newSnapshotQueue(1, 1, nil) + //h.SnapshotQueue = newSnapshotQueue(1, 1, nil) return h } // fragTestMustOpenIndex returns a new, opened index at a temporary path. Panic on error. -func fragTestMustOpenIndex(index string, holder *Holder, opt IndexOptions) *Index { - path, err := ioutil.TempDir(*TempDir, "pilosa-index-") - if err != nil { - panic(err) - } - holder.Path = path +func fragTestMustOpenIndex(holderDir, index string, holder *Holder, opt IndexOptions) *Index { + holder.Path = holderDir + holder.mu.Lock() idx, err := holder.createIndex(index, opt) + holder.mu.Unlock() panicOn(err) idx.keys = opt.Keys idx.trackExistence = opt.TrackExistence - if err := idx.Open(); err != nil { + if err := idx.Open(false); err != nil { panic(err) } return idx @@ -3213,26 +3693,28 @@ func fragTestMustOpenIndex(index string, holder *Holder, opt IndexOptions) *Inde // mustOpenFragment returns a new instance of Fragment with a temporary path. func mustOpenFragmentFlags(index, field, view string, shard uint64, cacheType string, flags byte) (*fragment, *Index) { - file, err := ioutil.TempFile(*TempDir, "pilosa-fragment-") - if err != nil { - panic(err) - } - file.Close() + + holderDir, err := ioutil.TempDir(*TempDir, "holder-dir") + panicOn(err) if cacheType == "" { cacheType = DefaultCacheType } - // new: th := newTestHolder() - idx := fragTestMustOpenIndex(index, th, IndexOptions{}) - f := newFragment(th, file.Name(), index, field, view, shard, flags) + idx := fragTestMustOpenIndex(holderDir, index, th, IndexOptions{}) + if th.NeedsSnapshot() { + th.SnapshotQueue = newSnapshotQueue(1, 1, nil) + } + + fragDir := fmt.Sprintf("%v/%v/views/%v/fragments/", idx.path, field, view) + panicOn(os.MkdirAll(fragDir, 0777)) + fragPath := fragDir + fmt.Sprintf("%v", shard) + f := newFragment(th, fragPath, index, field, view, shard, flags) tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f}) f.txTestingOnly = tx - //old: f := newFragment(testHolder, file.Name(), index, field, view, shard, flags) - f.CacheType = cacheType f.RowAttrStore = &memAttrStore{ store: make(map[uint64]map[string]interface{}), @@ -3292,9 +3774,9 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("firstContainer", func(t *testing.T) { f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") _ = idx - defer f.Clean(t) tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() + defer f.Clean(t) expectedAll := make([]uint64, 0) expectedOdd := make([]uint64, 0) @@ -3326,9 +3808,9 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("secondRow", func(t *testing.T) { f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") _ = idx - defer f.Clean(t) tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() + defer f.Clean(t) expected := []uint64{1, 2} if _, err := f.setBit(tx, 1, 66000); err != nil { @@ -3338,6 +3820,10 @@ func TestFragment_RowsIteration(t *testing.T) { } else if _, err := f.setBit(tx, 2, 166000); err != nil { t.Fatal(err) } + panicOn(tx.Commit()) + + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() ids, err := f.rows(context.Background(), tx, 0) if err != nil { @@ -3357,9 +3843,9 @@ func TestFragment_RowsIteration(t *testing.T) { t.Run("combinations", func(t *testing.T) { f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") _ = idx - defer f.Clean(t) tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() + defer f.Clean(t) expectedRows := make([]uint64, 0) for r := uint64(1); r < uint64(10000); r += 250 { @@ -3426,6 +3912,7 @@ func TestFragment_RoaringImport(t *testing.T) { if err != nil { t.Fatalf("importing roaring: %v", err) } + exp := calcExpected(test[:num+1]...) for row, expCols := range exp { cols := f.mustRow(tx, uint64(row)).Columns() @@ -3471,6 +3958,7 @@ func TestFragment_RoaringImportTopN(t *testing.T) { if err != nil { t.Fatalf("bulk importing ids: %v", err) } + expPairs := calcTop(test.rowIDs, test.colIDs) pairs, err := f.top(tx, topOptions{}) if err != nil { @@ -3602,9 +4090,9 @@ func TestFragmentRowIterator(t *testing.T) { t.Run("basic", func(t *testing.T) { f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) _ = idx - defer f.Clean(t) tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() + defer f.Clean(t) f.mustSetBits(tx, 0, 0) f.mustSetBits(tx, 1, 0) @@ -3650,7 +4138,7 @@ func TestFragmentRowIterator(t *testing.T) { _ = idx defer f.Clean(t) tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() f.mustSetBits(tx, 1, 0) f.mustSetBits(tx, 3, 0) @@ -3696,7 +4184,7 @@ func TestFragmentRowIterator(t *testing.T) { _ = idx defer f.Clean(t) tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() f.mustSetBits(tx, 0, 0) f.mustSetBits(tx, 1, 0) @@ -3727,11 +4215,13 @@ func TestFragmentRowIterator(t *testing.T) { }) t.Run("skipped rows wrapped", func(t *testing.T) { + skipForRBF(t) + f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) _ = idx defer f.Clean(t) tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() f.mustSetBits(tx, 1, 0) f.mustSetBits(tx, 3, 0) @@ -3762,7 +4252,192 @@ func TestFragmentRowIterator(t *testing.T) { }) } +// same, with commits +func TestFragmentRowIterator_WithTxCommit(t *testing.T) { + t.Run("basic", func(t *testing.T) { + f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + _ = idx + tx := f.txTestingOnly + defer tx.Rollback() + defer f.Clean(t) + + f.mustSetBits(tx, 0, 0) + f.mustSetBits(tx, 1, 0) + f.mustSetBits(tx, 2, 0) + f.mustSetBits(tx, 3, 0) + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + iter, err := f.rowIterator(tx, false) + if err != nil { + t.Fatal(err) + } + for i := uint64(0); i < 4; i++ { + row, id, _, wrapped, err := iter.Next() + if err != nil { + t.Fatal(err) + } + if id != i { + t.Fatalf("expected row %d but got %d", i, id) + } + if wrapped { + t.Fatalf("shouldn't have wrapped") + } + if !reflect.DeepEqual(row.Columns(), []uint64{0}) { + t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) + } + } + row, id, _, wrapped, err := iter.Next() + if err != nil { + t.Fatal(err) + } + if row != nil { + t.Fatalf("row should be nil after iterator is exhausted, got %v", row.Columns()) + } + if id != 0 { + t.Fatalf("id should be 0 after iterator is exhausted, got %d", id) + } + if !wrapped { + t.Fatalf("wrapped should be true after iterator is exhausted") + } + }) + + t.Run("skipped rows", func(t *testing.T) { + f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + _ = idx + defer f.Clean(t) + tx := f.txTestingOnly + defer tx.Rollback() + + f.mustSetBits(tx, 1, 0) + f.mustSetBits(tx, 3, 0) + f.mustSetBits(tx, 5, 0) + f.mustSetBits(tx, 7, 0) + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + iter, err := f.rowIterator(tx, false) + if err != nil { + t.Fatal(err) + } + for i := uint64(1); i < 8; i += 2 { + row, id, _, wrapped, err := iter.Next() + if err != nil { + t.Fatal(err) + } + if id != i { + t.Fatalf("expected row %d but got %d", i, id) + } + if wrapped { + t.Fatalf("shouldn't have wrapped") + } + if !reflect.DeepEqual(row.Columns(), []uint64{0}) { + t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) + } + } + row, id, _, wrapped, err := iter.Next() + if err != nil { + t.Fatal(err) + } + if row != nil { + t.Fatalf("row should be nil after iterator is exhausted, got %v", row.Columns()) + } + if id != 0 { + t.Fatalf("id should be 0 after iterator is exhausted, got %d", id) + } + if !wrapped { + t.Fatalf("wrapped should be true after iterator is exhausted") + } + }) + + t.Run("basic wrapped", func(t *testing.T) { + f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + _ = idx + defer f.Clean(t) + tx := f.txTestingOnly + defer tx.Rollback() + + f.mustSetBits(tx, 0, 0) + f.mustSetBits(tx, 1, 0) + f.mustSetBits(tx, 2, 0) + f.mustSetBits(tx, 3, 0) + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + iter, err := f.rowIterator(tx, true) + if err != nil { + t.Fatal(err) + } + for i := uint64(0); i < 5; i++ { + row, id, _, wrapped, err := iter.Next() + if err != nil { + t.Fatal(err) + } + if id != i%4 { + t.Fatalf("expected row %d but got %d", i%4, id) + } + if wrapped && i < 4 { + t.Fatalf("shouldn't have wrapped") + } else if !wrapped && i >= 4 { + t.Fatalf("should have wrapped") + } + if !reflect.DeepEqual(row.Columns(), []uint64{0}) { + t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) + } + } + }) + + t.Run("skipped rows wrapped", func(t *testing.T) { + f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeRanked) + _ = idx + defer f.Clean(t) + tx := f.txTestingOnly + defer tx.Rollback() + + f.mustSetBits(tx, 1, 0) + f.mustSetBits(tx, 3, 0) + f.mustSetBits(tx, 5, 0) + f.mustSetBits(tx, 7, 0) + + panicOn(tx.Commit()) + tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + defer tx.Rollback() + + iter, err := f.rowIterator(tx, true) + if err != nil { + t.Fatal(err) + } + for i := uint64(1); i < 10; i += 2 { + row, id, _, wrapped, err := iter.Next() + if err != nil { + t.Fatal(err) + } else if row == nil { + t.Fatal("expected row") + } + if id != i%8 { + t.Errorf("expected row %d but got %d", i%8, id) + } + if wrapped && i < 8 { + t.Errorf("shouldn't have wrapped") + } else if !wrapped && i >= 8 { + t.Errorf("should have wrapped") + } + if !reflect.DeepEqual(row.Columns(), []uint64{0}) { + t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) + } + } + }) +} + func TestUnionInPlaceMapped(t *testing.T) { + roaringOnlyTest(t) + f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) // note: clean has to be deferred first, because it has to run with // the lock *not* held, because it is sometimes so it has to grab the @@ -3928,7 +4603,7 @@ func TestIntLTRegression(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() _, err := f.setValue(tx, 1, 6, 33) if err != nil { @@ -3961,7 +4636,8 @@ func TestFragmentBSIUnsigned(t *testing.T) { defer f.Clean(t) // Obtain transaction. - tx := &RoaringTx{fragment: f} + tx := f.txTestingOnly + defer tx.Rollback() // Number of bits to test. const k = 6 @@ -4116,13 +4792,181 @@ func TestFragmentBSIUnsigned(t *testing.T) { }) } +// same, WithTxCommit version +func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) { + f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) + _ = idx + defer f.Clean(t) + + // Obtain transaction. + tx := f.txTestingOnly + defer tx.Rollback() + + // Number of bits to test. + const k = 6 + + // Load all numbers into an effectively diagonal matrix. + for i := 0; i < 1<", func(t *testing.T) { + for i := minCheck; i < maxCheck; i++ { + row, err := f.rangeGT(tx, k, int64(i), false) + if err != nil { + t.Fatalf("failed to query fragment: %v", err) + } + var expect []uint64 + switch { + case i < 0: + expect = cols + case i < len(cols)-1: + expect = cols[i+1:] + default: + } + got := row.Columns() + if !sliceEq(expect, got) { + t.Errorf("expected %v but got %v for x > %d", expect, got, i) + } + } + }) + t.Run(">=", func(t *testing.T) { + for i := minCheck; i < maxCheck; i++ { + row, err := f.rangeGT(tx, k, int64(i), true) + if err != nil { + t.Fatalf("failed to query fragment: %v", err) + } + var expect []uint64 + switch { + case i < 0: + expect = cols + case i < len(cols): + expect = cols[i:] + default: + } + got := row.Columns() + if !sliceEq(expect, got) { + t.Errorf("expected %v but got %v for x >= %d", expect, got, i) + } + } + }) + t.Run("Range", func(t *testing.T) { + for i := minCheck; i < maxCheck; i++ { + for j := i; j < maxCheck; j++ { + row, err := f.rangeBetween(tx, k, int64(i), int64(j)) + if err != nil { + t.Fatalf("failed to query fragment: %v", err) + } + var lower, upper int + switch { + case i < 0: + lower = 0 + case i > len(cols): + lower = len(cols) + default: + lower = i + } + switch { + case j < 0: + upper = 0 + case j >= len(cols): + upper = len(cols) + default: + upper = j + 1 + } + expect := cols[lower:upper] + got := row.Columns() + if !sliceEq(expect, got) { + t.Errorf("expected %v but got %v for %d <= x <= %d", expect, got, i, j) + } + } + } + }) + t.Run("==", func(t *testing.T) { + for i := minCheck; i < maxCheck; i++ { + row, err := f.rangeEQ(tx, k, int64(i)) + if err != nil { + t.Fatalf("failed to query fragment: %v", err) + } + var expect []uint64 + if i >= 0 && i < len(cols) { + expect = cols[i : i+1] + } + got := row.Columns() + if !sliceEq(expect, got) { + t.Errorf("expected %v but got %v for x == %d", expect, got, i) + } + } + }) +} + func TestFragmentBSISigned(t *testing.T) { f, idx := mustOpenFragment("i", "f", "v", 0, CacheTypeNone) _ = idx defer f.Clean(t) // Obtain transaction. - tx := &RoaringTx{fragment: f} + tx := f.txTestingOnly + defer tx.Rollback() // Number of bits to test. const k = 6 @@ -4279,6 +5123,8 @@ func TestFragmentBSISigned(t *testing.T) { } func TestImportClearRestart(t *testing.T) { + roaringOnlyTest(t) + tests := []struct { rows []uint64 cols []uint64 @@ -4365,7 +5211,18 @@ func TestImportClearRestart(t *testing.T) { check(t, tx, f, exp) - f2 := newFragment(NewHolder(DefaultPartitionN), f.path, "i", "f", viewStandard, 0, 0) + h := NewHolder(DefaultPartitionN) + h.Path = filepath.Dir(f.path) + idx2, err := h.CreateIndex("i", IndexOptions{}) + _ = idx2 + panicOn(err) + + // OVERWRITING the f.path with a new fragment + f2 := newFragment(h, f.path, "i", "f", viewStandard, 0, 0) + + // f2, idx2 := mustOpenFragment("i", "f", viewStandard, 0, "") + // _ = idx2 + f2.MaxOpN = maxOpN f2.CacheType = f.CacheType @@ -4408,7 +5265,13 @@ func TestImportClearRestart(t *testing.T) { panicOn(tx2.Commit()) - f3 := newFragment(NewHolder(DefaultPartitionN), f2.path, "i", "f", viewStandard, 0, 0) + h3 := NewHolder(DefaultPartitionN) + h3.Path = filepath.Dir(f2.path) + idx3, err := h3.CreateIndex("i", IndexOptions{}) + _ = idx3 + panicOn(err) + + f3 := newFragment(h3, f2.path, "i", "f", viewStandard, 0, 0) f3.MaxOpN = maxOpN f3.CacheType = f.CacheType @@ -4422,7 +5285,6 @@ func TestImportClearRestart(t *testing.T) { err = f3.Open() if err != nil { - // TODO(jea): might be a flaky test? when run from make test t.Fatalf("opening f3: %v", err) } defer f3.Clean(t) @@ -4464,6 +5326,7 @@ func check(t *testing.T, tx Tx, f *fragment, exp map[uint64]map[uint64]struct{}) } func TestImportValueConcurrent(t *testing.T) { + skipForRBF(t) f, idx := mustOpenBSIFragment("i", "f", viewBSIGroupPrefix+"foo", 0) switch idx.Txf.TxType() { @@ -4529,13 +5392,18 @@ func TestImportMultipleValues(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() err := f.importValue(tx, test.cols, test.vals, test.depth, false) if err != nil { t.Fatalf("importing values: %v", err) } + // probably too slow, would hit disk alot: + //panicOn(tx.Commit()) + //tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f}) + //defer tx.Rollback() + for i := range test.checkCols { cc, cv := test.checkCols[i], test.checkVals[i] n, exists, err := f.value(tx, cc, test.depth) @@ -4592,7 +5460,7 @@ func TestImportValueRowCache(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // First import (tc1) if err := f.importValue(tx, test.tc1.cols, test.tc1.vals, test.tc1.depth, false); err != nil { @@ -4625,12 +5493,13 @@ func TestFragmentConcurrentReadWrite(t *testing.T) { _ = idx defer f.Clean(t) + // Obtain transaction, but don't start another b/c the + // two goroutines below need the same view. + tx := f.txTestingOnly + defer tx.Rollback() + eg := &errgroup.Group{} eg.Go(func() error { - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME - for i := uint64(0); i < 1000; i++ { _, err := f.setBit(tx, i%4, i) if err != nil { @@ -4640,10 +5509,6 @@ func TestFragmentConcurrentReadWrite(t *testing.T) { return nil }) - // Obtain transaction. - tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME - acc := uint64(0) for i := uint64(0); i < 100; i++ { r := f.mustRow(tx, i%4) @@ -4659,6 +5524,7 @@ func TestFragmentConcurrentReadWrite(t *testing.T) { func TestRemapCache(t *testing.T) { f, idx := mustOpenFragment("i", "f", viewStandard, 0, "") _ = idx + index, field, view, shard := f.index, f.field, f.view, f.shard // request a panic that doesn't kill the program on fault wouldFault := debug.SetPanicOnFault(true) @@ -4678,10 +5544,10 @@ func TestRemapCache(t *testing.T) { // Obtain transaction. tx := f.txTestingOnly - defer tx.Rollback() //LOOKATME + defer tx.Rollback() // create a container - _, err := f.storage.Add(65537) + _, err := tx.Add(index, field, view, shard, !doBatched, 65537) if err != nil { t.Fatalf("storage add: %v", err) } @@ -4694,7 +5560,7 @@ func TestRemapCache(t *testing.T) { _ = f.mustRow(tx, 0) // add a bit that isn't in that container, so that container doesn't // change - _, err = f.storage.Add(2) + _, err = tx.Add(index, field, view, shard, !doBatched, 2) if err != nil { t.Fatalf("storage add: %v", err) } @@ -4786,3 +5652,9 @@ func TestFragment_Bug_Q2DoubleDelete(t *testing.T) { t.Fatalf("expected nothing got %v", res) } } + +func skipForRBF(tb testing.TB) { + if os.Getenv("PILOSA_TXSRC") == "rbf" { + tb.Skip("skip for RBF") + } +} diff --git a/generation.go b/generation.go index e9c4e93c5..268585b1d 100644 --- a/generation.go +++ b/generation.go @@ -261,6 +261,7 @@ func (m *mmapGeneration) openFile() (shouldClose bool, err error) { if err != nil { return false, err } + // do we actually want this in every openFile? I don't know. if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { _ = syswrap.CloseFile(m.file) @@ -438,3 +439,25 @@ func newGeneration(existing generation, path string, readData bool, setup func([ // does get cleaned up. return &m, nil } + +// NopGeneration is used in fragment.openStorage() to short-circuit +// generation stuff that only applies to RoaringTx; doesn't apply to RBFTx/BadgerTx/etc. +type NopGeneration struct { +} + +func (g *NopGeneration) Transaction(w *io.Writer, f func() error) error { + return f() +} +func (g *NopGeneration) Done() {} +func (g *NopGeneration) Generation() int64 { + return 0 +} +func (g *NopGeneration) ID() string { + return "NOP" +} +func (g *NopGeneration) Dead() bool { + return true +} +func (g *NopGeneration) Bytes() (ret []byte) { + return +} diff --git a/generation_test.go b/generation_test.go index 7df71fc98..bc0a1854f 100644 --- a/generation_test.go +++ b/generation_test.go @@ -53,8 +53,11 @@ func TestGenerationPanic(t *testing.T) { if unsafe.Pointer(&prevData[0]) == unsafe.Pointer(&newData[0]) { t.Fatalf("test can't run usefully, didn't get new data pointer") } - - err := f.gen.Transaction(&f.storage.OpWriter, func() error { + var wp *io.Writer + if f.storage != nil { + wp = &f.storage.OpWriter + } + err := f.gen.Transaction(wp, func() error { prevData[0] = 0x3c return nil }) diff --git a/go.mod b/go.mod index acc83bb52..769fd50b9 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,8 @@ require ( github.com/benbjohnson/immutable v0.2.0 github.com/boltdb/bolt v1.3.1 github.com/cespare/xxhash v1.1.0 - github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 - github.com/dchest/blake2b v1.0.0 // indirect - github.com/dgraph-io/badger v1.6.1-0.20191025180844-32a2548a9d85 // indirect github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.0 @@ -22,8 +19,7 @@ require ( github.com/gorilla/handlers v1.3.0 github.com/gorilla/mux v1.7.0 github.com/hashicorp/memberlist v0.1.3 - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2 + github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2 // indirect github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 github.com/opentracing/opentracing-go v1.1.0 github.com/pelletier/go-toml v1.2.0 @@ -40,7 +36,6 @@ require ( github.com/uber-go/atomic v1.4.0 // indirect github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible // indirect - github.com/willoch/tago v0.0.0-20180311150625-8f2f8e8900dc // indirect github.com/zeebo/blake3 v0.0.4 go.uber.org/atomic v1.4.0 // indirect golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect diff --git a/go.sum b/go.sum index fbed4774f..5d09bde2d 100644 --- a/go.sum +++ b/go.sum @@ -26,9 +26,6 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= @@ -40,18 +37,8 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dchest/blake2b v1.0.0 h1:KK9LimVmE0MjRl9095XJmKqZ+iLxWATvlcpVFRtaw6s= -github.com/dchest/blake2b v1.0.0/go.mod h1:U034kXgbJpCle2wSk5ybGIVhOSHCVLMDqOzcPEA0F7s= -github.com/dgraph-io/badger v1.6.1-0.20191025180844-32a2548a9d85 h1:oEqDRoxpep5ZlTxrAFc2yg+f0uBdUtkZE0uWsOru5bc= -github.com/dgraph-io/badger v1.6.1-0.20191025180844-32a2548a9d85/go.mod h1:cEjdIw+iaGXuQdsDymXPRcpp8yHXZ6PmwmDJajnVyJc= -github.com/dgraph-io/badger v1.6.1 h1:w9pSFNSdq/JPM1N12Fz/F/bzo993Is1W+Q7HjPzi7yg= github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 h1:JBNM90aGLCiF9iJYvpvayMpYeW498v5ZDZqE2chqZ2A= github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= -github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl6XJI= -github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= -github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs= -github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU= -github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= @@ -115,8 +102,10 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -145,7 +134,6 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU= github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -180,29 +168,25 @@ github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38= -github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= @@ -211,15 +195,13 @@ github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/willoch/tago v0.0.0-20180311150625-8f2f8e8900dc h1:Jsemerl8qK30jGNdYlxGZpZk9RjB4pqvezJgxqUgy30= -github.com/willoch/tago v0.0.0-20180311150625-8f2f8e8900dc/go.mod h1:9WHA/f8A/TRK+WQQZhqx47In4pnIhMTH6UrsgqqgsVQ= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0= +github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/blake3 v0.0.4-0.20200428182842-252974700486 h1:yh0zEy8it58x/IPNtKKuvKUkxSIaq4s5XiRSd40JuYs= -github.com/zeebo/blake3 v0.0.4-0.20200428182842-252974700486/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34= github.com/zeebo/blake3 v0.0.4 h1:vtZ4X8B2lKXZFg2Xyg6Wo36mvmnJvc2VQYTtA4RDCkI= github.com/zeebo/blake3 v0.0.4/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34= +github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05 h1:4pW5fMvVkrgkMXdvIsVRRTs69DWYA8uNNQsu1stfVKU= github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05/go.mod h1:Gr+78ptB0MwXxm//LBaEvBiaXY7hXJ6KGe2V32X2F6E= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -243,8 +225,6 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU= -golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -263,8 +243,6 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8= -golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= @@ -292,6 +270,7 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= @@ -302,4 +281,3 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130= diff --git a/holder.go b/holder.go index 58dca7f2c..e74672d31 100644 --- a/holder.go +++ b/holder.go @@ -528,9 +528,9 @@ func (h *Holder) Open() error { if h.isCoordinator() { index.createdAt = timestamp() - err = index.OpenWithTimestamp() + err = index.OpenWithTimestamp(false) } else { - err = index.Open() + err = index.Open(false) } if err != nil { if err == ErrName { @@ -626,6 +626,17 @@ func (h *Holder) BeginTx(writable bool, index *Index) (Tx, error) { return index.Txf.NewTx(Txo{Write: writable, Index: index}), nil } +func (h *Holder) NeedsSnapshot() bool { + h.mu.RLock() + defer h.mu.RUnlock() + for _, idx := range h.indexes { + if idx.NeedsSnapshot() { + return true + } + } + return false +} + // HasData returns true if Holder contains at least one index. // This is used to determine if the rebalancing of data is necessary // when a node joins the cluster. @@ -802,7 +813,20 @@ func (h *Holder) applyCreatedAt(indexes []*IndexInfo) { } // IndexPath returns the path where a given index is stored. -func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) } +func (h *Holder) IndexPath(name string) string { + return filepath.Join(h.Path, name) +} + +// HolderPathFromIndexPath is +// used by test/index.go:71 in test.Index.Reopen() to get the right +// path into a test Holder that doesn't know its own proper path. +// If the Holder changes index paths to being something other than +// holderPath + "/" + indexName, this will need adjusting too. +func (h *Holder) HolderPathFromIndexPath(indexPath, indexName string) string { + n := len(indexPath) + hpath2 := indexPath[:n-(len(indexName)+1)] + return hpath2 +} // Index returns the index by name. func (h *Holder) Index(name string) *Index { @@ -811,7 +835,9 @@ func (h *Holder) Index(name string) *Index { return h.index(name) } -func (h *Holder) index(name string) *Index { return h.indexes[name] } +func (h *Holder) index(name string) *Index { + return h.indexes[name] +} // Indexes returns a list of all indexes in the holder. func (h *Holder) Indexes() []*Index { @@ -867,7 +893,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { index.keys = opt.Keys index.trackExistence = opt.TrackExistence - if err = index.Open(); err != nil { + if err = index.Open(true); err != nil { return nil, errors.Wrap(err, "opening") } if err = index.saveMeta(); err != nil { @@ -1799,3 +1825,15 @@ func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) { } return nil } + +// used by Index.openFields(), enabling Tx / Txf by telling +// the holder about its own indexes. +func (h *Holder) addIndexFromField(idx *Index) { + h.mu.Lock() + h.indexes[idx.Name()] = idx + h.mu.Unlock() +} + +func (h *Holder) unprotectedAddIndexFromField(idx *Index) { + h.indexes[idx.Name()] = idx +} diff --git a/holder_test.go b/holder_test.go index a527307ba..08b84acc6 100644 --- a/holder_test.go +++ b/holder_test.go @@ -32,6 +32,7 @@ import ( ) func TestHolder_Open(t *testing.T) { + t.Run("ErrIndexName", func(t *testing.T) { h := test.MustOpenHolder() @@ -165,6 +166,8 @@ func TestHolder_Open(t *testing.T) { }) t.Run("ErrFragmentStoragePermission", func(t *testing.T) { + roaringOnlyTest(t) + if os.Geteuid() == 0 { t.Skip("Skipping permissions test since user is root.") } @@ -201,6 +204,8 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) { + roaringOnlyTest(t) + h := test.MustOpenHolder() defer h.Close() @@ -233,6 +238,8 @@ func TestHolder_Open(t *testing.T) { } }) t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) { + roaringOnlyTest(t) + h := test.MustOpenHolder() defer h.Close() @@ -397,6 +404,8 @@ func TestHolder_HasData(t *testing.T) { // Ensure holder can delete an index and its underlying files. func TestHolder_DeleteIndex(t *testing.T) { + skipForRBF(t) + hldr := test.MustOpenHolder() defer hldr.Close() @@ -572,7 +581,6 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) { // Leave the third replica empty to force a block merge. // - err = c[0].Server.SyncData() if err != nil { t.Fatalf("syncing node 0: %v", err) @@ -692,6 +700,8 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) { // Ensure holder can sync integer views with a remote holder. func TestHolderSyncer_IntField(t *testing.T) { + skipForRBF(t) + t.Run("BasicSync", func(t *testing.T) { c := test.MustNewCluster(t, 2) c[0].Config.Cluster.ReplicaN = 2 @@ -723,24 +733,17 @@ func TestHolderSyncer_IntField(t *testing.T) { hldr0.SetValue("i", "f", 1, 1) // in c0 expect the 1 bit - //idx0.Dump("in c0, before SyncData") // Set data on node1. columnID=2, value=2 idx1 := hldr1.SetValue("i", "f", 2, 2) _ = idx1 - //idx1.Dump("in c1, before SyncData") - - //vv("before c[0] SyncData") err = c[0].Server.SyncData() if err != nil { t.Fatalf("syncing node 0: %v", err) } - //vv("after c[0] SyncData") // expect 3 rows, the 1 bit + 2 rows for the 2 value as BSI. But, we only see that c0 overwrote c1. - //idx0.Dump("in c0, after syncData") - //idx1.Dump("in c1, after syncData") // Problem is: data at c1 was replaced by c0, instead of being merged with existing c1. // Problem is: data at c0 did not receive and merge the c1 data. @@ -810,8 +813,6 @@ func TestHolderSyncer_IntField(t *testing.T) { } // dump the badger keys for both c0 and c1 - //vv("in c0, allkeys = '%v'", idx0.StringifiedBadgerKeys(nil)) - //vv("in c1, allkeys = '%v'", c[1].index.StringifiedBadgerKeys()) // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { diff --git a/index.go b/index.go index 1474616bc..d1a7770df 100644 --- a/index.go +++ b/index.go @@ -72,8 +72,21 @@ type Index struct { Txf *TxFactory } -// NewIndex returns a new instance of Index. +// OpenIndex opens or starts a new Index on path. Path +// can be empty. +func OpenIndex(holder *Holder, path, name string) (*Index, error) { + openExisting := true + return openOrCreateNewIndex(holder, path, name, openExisting) +} + +// NewIndex returns a new instance of Index at path. It will erase anything +// old already in path. func NewIndex(holder *Holder, path, name string) (*Index, error) { + openExisting := false + return openOrCreateNewIndex(holder, path, name, openExisting) +} + +func openOrCreateNewIndex(holder *Holder, path, name string, openExisting bool) (*Index, error) { // Emulate what the spf13/cobra does, letting env vars override // the defaults, because we may be under a simple "go test" run where @@ -99,16 +112,16 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) { } } - txf, err := newTxFactory(txsrc, path) - if err != nil { - return nil, errors.Wrap(err, "creating newTxFactory") - } - - err = validateName(name) + err := validateName(name) if err != nil { return nil, errors.Wrap(err, "validating name") } + txf, err := NewTxFactory(txsrc, holder.Path, name, openExisting) + if err != nil { + return nil, errors.Wrap(err, "creating newTxFactory") + } + idx := &Index{ path: path, name: name, @@ -134,6 +147,14 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) { return idx, nil } +func (i *Index) NewTx(txo Txo) Tx { + return i.Txf.NewTx(txo) +} + +func (i *Index) NeedsSnapshot() bool { + return i.Txf.NeedsSnapshot() +} + // CreatedAt is an timestamp for a specific version of an index. func (i *Index) CreatedAt() int64 { i.mu.RLock() @@ -148,7 +169,9 @@ func (i *Index) Name() string { return i.name } func (i *Index) QualifiedName() string { return i.qualifiedName } // Path returns the path the index was initialized with. -func (i *Index) Path() string { return i.path } +func (i *Index) Path() string { + return i.path +} // TranslateStorePath returns the translation database path for a partition. func (i *Index) TranslateStorePath(partitionID int) string { @@ -181,12 +204,12 @@ func (i *Index) options() IndexOptions { } // Open opens and initializes the index. -func (i *Index) Open() error { return i.open(false) } +func (i *Index) Open(haveHolderLock bool) error { return i.open(false, haveHolderLock) } // OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields. -func (i *Index) OpenWithTimestamp() error { return i.open(true) } +func (i *Index) OpenWithTimestamp(haveHolderLock bool) error { return i.open(true, haveHolderLock) } -func (i *Index) open(withTimestamp bool) (err error) { +func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) { // Ensure the path exists. i.holder.Logger.Debugf("ensure index path exists: %s", i.path) if err := os.MkdirAll(i.path, 0777); err != nil { @@ -200,7 +223,7 @@ func (i *Index) open(withTimestamp bool) (err error) { } i.holder.Logger.Debugf("open fields for index: %s", i.name) - if err := i.openFields(withTimestamp); err != nil { + if err := i.openFields(withTimestamp, haveHolderLock); err != nil { return errors.Wrap(err, "opening fields") } @@ -243,7 +266,7 @@ func (i *Index) open(withTimestamp bool) (err error) { var indexQueue = make(chan struct{}, 8) // openFields opens and initializes the fields inside the index. -func (i *Index) openFields(withTimestamp bool) error { +func (i *Index) openFields(withTimestamp, haveHolderLock bool) error { f, err := os.Open(i.path) if err != nil { return errors.Wrap(err, "opening directory") @@ -273,7 +296,31 @@ fileLoop: <-indexQueue }() i.holder.Logger.Debugf("open field: %s", fi.Name()) + mu.Lock() + + // i.holder needs to know about its index i for the Txf to work. + // + // We face either a deadlock or a race here. + // + // We get a deadlock in TestIndex_CreateField/"BSIFields"/"OK" + // if we call addIndexFromField, because in that test + // we get here while already holding i.holder.mu. + // + // On the other had, we get races on other tests + // such as TestExecutor_Execute_Existence/Row + // if we call unprotectedAddIndexFromField which does + // not lock i.holder.mu. + // + // The resolution was to have the goroutines that are holding + // the lock already tell us. That is the haveHolderLock + // argument. + if haveHolderLock { + i.holder.unprotectedAddIndexFromField(i) + } else { + i.holder.addIndexFromField(i) + } + fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if withTimestamp { fld.createdAt = timestamp() @@ -287,6 +334,7 @@ fileLoop: // up a foreign index. fld.holder = i.holder + // open all the views if err := fld.Open(); err != nil { return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err) } @@ -546,6 +594,9 @@ func (i *Index) createField(name string, opt *FieldOptions) (*Field, error) { // Add to index's field lookup. i.fields[name] = f + // enable Txf to find the index in field_test.go TestField_SetValue + f.idx = i + // Kick off the field's translation sync process. if err := i.translationSyncer.Reset(); err != nil { return nil, errors.Wrap(err, "resetting translation syncer") @@ -559,6 +610,7 @@ func (i *Index) newField(path, name string) (*Field, error) { if err != nil { return nil, err } + f.idx = i f.Stats = i.Stats f.broadcaster = i.broadcaster f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data")) @@ -582,9 +634,8 @@ func (i *Index) DeleteField(name string) error { return errors.Wrap(err, "closing") } - // Delete field directory. - if err := os.RemoveAll(i.fieldPath(name)); err != nil { - return errors.Wrap(err, "removing directory") + if err := i.Txf.DeleteFieldFromStore(i.name, name, i.fieldPath(name)); err != nil { + return errors.Wrap(err, "Txf.DeleteFieldFromStore") } // If the field being deleted is the existence field, @@ -661,3 +712,13 @@ type importValueData struct { func FormatQualifiedIndexName(index string) string { return fmt.Sprintf("%s\x00", index) } + +// Dump prints to stdout the contents of the roaring Containers +// stored in idx. Mostly for debugging. +func (idx *Index) Dump(label string) { + fileline := FileLine(2) + tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx}) + defer tx.Rollback() + fmt.Printf("\n%v Index.Dump('%v') for index '%v':\n", fileline, label, idx.name) + tx.Dump() +} diff --git a/index_internal_test.go b/index_internal_test.go index 5b5a5b5a3..4f569c745 100644 --- a/index_internal_test.go +++ b/index_internal_test.go @@ -25,7 +25,10 @@ func mustOpenIndex(opt IndexOptions) *Index { if err != nil { panic(err) } - index, err := NewIndex(NewHolder(1), path, "i") + h := NewHolder(1) + h.Path = path + index, err := h.CreateIndex("i", opt) + if err != nil { panic(err) } @@ -33,7 +36,7 @@ func mustOpenIndex(opt IndexOptions) *Index { index.keys = opt.Keys index.trackExistence = opt.TrackExistence - if err := index.Open(); err != nil { + if err := index.Open(false); err != nil { panic(err) } return index @@ -44,7 +47,7 @@ func (i *Index) reopen() error { if err := i.Close(); err != nil { return err } - if err := i.Open(); err != nil { + if err := i.Open(false); err != nil { return err } return nil diff --git a/license.exceptions b/license.exceptions index e703b7c2a..ace441ae6 100644 --- a/license.exceptions +++ b/license.exceptions @@ -10,3 +10,5 @@ ./logger/filewriter.go ./logger/filewriter_test.go ./vprint.go +./rbf/vprint.go +./cmd/loader/vprint.go diff --git a/like_test.go b/like_test.go index 1382901ec..7e5268dea 100644 --- a/like_test.go +++ b/like_test.go @@ -20,7 +20,6 @@ import ( ) func TestPlanLike(t *testing.T) { - t.Parallel() cases := []struct { name string diff --git a/mmap_test.go b/mmap_test.go index 20fbd7685..ab9643c73 100644 --- a/mmap_test.go +++ b/mmap_test.go @@ -32,11 +32,11 @@ type cv struct { func forceSnapshotsCheckMapping(t *testing.T) { depth := uint(6) f, idx := mustOpenBSIFragment("i", "f", viewStandard, 0) - _ = idx f.Logger = logger.NewLogfLogger(t) defer f.Clean(t) - tx := &RoaringTx{fragment: f} + tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f}) + defer tx.Rollback() for i := 0; i < f.MaxOpN; i++ { _, _ = f.setBit(tx, 0, uint64(32*i)) @@ -86,6 +86,8 @@ func forceSnapshotsCheckMapping(t *testing.T) { // in newGeneration in generation.go. So this is probably useless but it's // a failure mode we've been bitten by once... func TestMmapBehavior(t *testing.T) { + skipForRBF(t) + var changed bool var original uint64 defer func() { diff --git a/pilosa_test.go b/pilosa_test.go index 59b6fe082..55a5bbd65 100644 --- a/pilosa_test.go +++ b/pilosa_test.go @@ -15,6 +15,7 @@ package pilosa_test import ( + "os" "strings" "testing" @@ -55,3 +56,9 @@ func TestAddressWithDefaults(t *testing.T) { } } } + +func skipForRBF(tb testing.TB) { + if os.Getenv("PILOSA_TXSRC") == "rbf" { + tb.Skip("skip for RBF") + } +} diff --git a/pql/ast.go b/pql/ast.go index 8ebfaf122..b150cd3c5 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -22,8 +22,6 @@ import ( "strconv" "strings" "time" - - "github.com/molecula/ext" ) // Query represents a PQL query. @@ -345,7 +343,8 @@ var callInfoByFunc = map[string]callInfo{ "Row": {allowUnknown: true}, "Range": {allowUnknown: true}, - "Distinct": {allowUnknown: true}, + "Distinct": {allowUnknown: true, callType: PrecallGlobal}, + "Condition": {allowUnknown: true}, // allow only "field=X" cases with string field names "Max": allowField, @@ -462,30 +461,6 @@ var callInfoByFunc = map[string]callInfo{ }, } -// RegisterPluginFuncs adds arg validation for plugin funcs. Not very good -// arg validation. -func RegisterPluginFuncs(ops []ext.BitmapOp) { - for _, op := range ops { - // ignore overlap for now. This should change. - if _, ok := callInfoByFunc[op.Name]; ok { - continue - } - ci := callInfo{allowUnknown: true} - if len(op.Reserved) > 0 { - // mark these as valid/known reserved words - ci.prototypes = make(map[string]interface{}) - for _, res := range op.Reserved { - ci.prototypes[res] = nil - } - } - t := op.Func.BitmapOpType() - if t.Precall == ext.OpPrecallGlobal { - ci.callType = PrecallGlobal - } - callInfoByFunc[op.Name] = ci - } -} - // CheckCallInfo tries to validate that arguments are correct and valid for the // given call. It does not guarantee checking all possible errors; for instance, // if an argument is a field name, CheckCallInfo can't validate that the field @@ -505,6 +480,11 @@ func (c *Call) CheckCallInfo() error { if !ok && strings.HasPrefix(k, "_") { return fmt.Errorf("'%s': unknown reserved arg '%s'", c.String(), k) } + if call, ok := v.(*Call); ok { + if err := call.CheckCallInfo(); err != nil { + return err + } + } if acceptable == nil { continue } diff --git a/rbf/blake3.go b/rbf/blake3.go new file mode 100644 index 000000000..3935d2576 --- /dev/null +++ b/rbf/blake3.go @@ -0,0 +1,97 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rbf + +import ( + "encoding/binary" + "fmt" + "sync" + + cryptorand "crypto/rand" + "github.com/zeebo/blake3" +) + +// Blake3Hasher is a thread/goroutine safe way to +// obtain a blake3 cryptographic hash of input []byte. +// Reference https://github.com/BLAKE3-team/BLAKE3 +// suggests it is 6x faster than BLAKE2B. +// The Go github.com/zeebo/blake3 version is +// AVX2 and SSE4.1 accelerated. +type Blake3Hasher struct { + hasher *blake3.Hasher + hasherMu sync.Mutex +} + +// NewBlake3Hasher returns a new Blake3Hasher. +func NewBlake3Hasher() *Blake3Hasher { + return &Blake3Hasher{ + hasher: blake3.New(), + } +} + +// CryptoHash writes the blake3 cryptographic hash of +// input into buffer and returns it. +// Like the standard libary's hash.Hash interface's Sum() method, +// the buffer is re-used and overwritten +// to avoid allocation. The caller determines the byte length of +// the outputCryptohash by the size of the supplied buffer +// slice, and this will be exactly equal to the supplies bytes. +// In this way, shorter or longer hashes can be provided as +// needed. +func (w *Blake3Hasher) CryptoHash(input []byte, buffer []byte) (outputCryptohash []byte) { + w.hasherMu.Lock() + w.hasher.Reset() + + // "Write implements part of the hash.Hash interface. It never returns an error." + // -- https://godoc.org/github.com/zeebo/blake3#Hasher.Write + _, _ = w.hasher.Write(input) + + // Digest.Read reads data from the hasher into buffer. + // "It always fills the entire buffer and never errors." + // -- https://godoc.org/github.com/zeebo/blake3#Digest + _, _ = w.hasher.Digest().Read(buffer) + + // no chance of panic, so avoid any defer cost. + w.hasherMu.Unlock() + + return buffer +} + +// blake3sum16 might be slower because we allocate a new hasher every time, but +// it is more conenient for writing debug code. It returns +// a 16 byte hash as a hexidecimal string. +func blake3sum16(input []byte) string { + hasher := blake3.New() + + _, _ = hasher.Write(input) + var buf [16]byte + _, _ = hasher.Digest().Read(buf[0:]) + + return fmt.Sprintf("%x", buf) +} + +// cryptoRandInt64 uses crypto/rand to get an random int64 +func cryptoRandInt64() int64 { + c := 8 + b := make([]byte, c) + _, err := cryptorand.Read(b) + if err != nil { + panic(err) + } + r := int64(binary.LittleEndian.Uint64(b)) + return r +} + +var _ = cryptoRandInt64 // happy linter diff --git a/rbf/cursor.go b/rbf/cursor.go index 9bb59fb6b..ed0a9f849 100644 --- a/rbf/cursor.go +++ b/rbf/cursor.go @@ -84,7 +84,7 @@ func runAdd(runs []roaring.Interval16, v uint16) ([]roaring.Interval16, bool) { } return runs, true } -func checkRun(runs []roaring.Interval16, key uint64) leafCell { +func checkRun(runs []roaring.Interval16, bitN int, key uint64) leafCell { if len(runs) >= RLEMaxSize { //convertToBitmap bitmap := make([]uint64, BitmapN) @@ -123,9 +123,9 @@ func checkRun(runs []roaring.Interval16, key uint64) leafCell { n += popcount(v) } - return leafCell{Key: key, N: int(n), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)} + return leafCell{Key: key, N: int(n), BitN: int(n), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)} } - return leafCell{Key: key, N: len(runs), Type: ContainerTypeRLE, Data: fromInterval16(runs)} + return leafCell{Key: key, N: len(runs), BitN: int(bitN + 1), Type: ContainerTypeRLE, Data: fromInterval16(runs)} } // Add sets a bit on the underlying bitmap. @@ -136,7 +136,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) { if exact, err := c.Seek(hi); err != nil { return false, err } else if !exact { - return true, c.putLeafCell(leafCell{Key: hi, Type: ContainerTypeArray, N: 1, Data: fromArray16([]uint16{lo})}) + return true, c.putLeafCell(leafCell{Key: hi, Type: ContainerTypeArray, N: 1, BitN: 1, Data: fromArray16([]uint16{lo})}) } // If the container exists and bit is not set then update the page. @@ -155,7 +155,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) { copy(other, a[:i]) other[i] = lo copy(other[i+1:], a[i:]) - return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), Data: fromArray16(other)}) + return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), BitN: cell.BitN + 1, Data: fromArray16(other)}) case ContainerTypeRLE: runs := toInterval16(cell.Data) @@ -163,7 +163,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) { copy(c.rle[:], runs) run, added := runAdd(c.rle[:len(runs)], lo) if added { - leaf := checkRun(run, cell.Key) + leaf := checkRun(run, cell.BitN, cell.Key) return true, c.putLeafCell(leaf) } return false, nil @@ -185,6 +185,8 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) { if err := c.tx.writeBitmapPage(pgno, fromArray64(a)); err != nil { return false, err } + // TODO(bbj): Update parent cell with new BitN. + return true, nil default: return false, fmt.Errorf("rbf.Cursor.Add(): invalid container type: %d", cell.Type) @@ -220,7 +222,7 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) { other := make([]uint16, len(a)-1) copy(other[:i], a[:i]) copy(other[i:], a[i+1:]) - return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), Data: fromArray16(other)}) + return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), BitN: cell.BitN - 1, Data: fromArray16(other)}) case ContainerTypeRLE: r := toInterval16(cell.Data) @@ -265,6 +267,8 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) { if err := c.tx.writeBitmapPage(pgno, fromArray64(a)); err != nil { return false, err } + + // TODO(bbj): Update parent cell to decrement BitN. return true, nil default: return false, fmt.Errorf("rbf.Cursor.Add(): invalid container type: %d", cell.Type) @@ -356,7 +360,10 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) { } in.Data = fromArray64(a) cell.Type = ContainerTypeBitmapPtr - bitmapPgno, _ := c.tx.allocate() + bitmapPgno, err := c.tx.allocate() + if err != nil { + return err + } cell.Data = fromPgno(bitmapPgno) } @@ -812,6 +819,7 @@ func (c *Cursor) Seek(key uint64) (exact bool, err error) { if err != nil { return false, err } + switch typ := readFlags(buf); typ { case PageTypeBranch: n := readCellN(buf) @@ -1089,6 +1097,7 @@ func (c *Cursor) goNextPage() error { func ConvertToLeafArgs(key uint64, c *roaring.Container) (result leafCell) { result.Key = key result.N = int(c.N()) + result.BitN = int(c.N()) result.Type = ContainerTypeNone if c.N() == 0 { return @@ -1136,7 +1145,7 @@ func (c *Cursor) merge(key uint64, data *roaring.Container) (bool, error) { if err != nil { return false, errors.Wrap(err, "cursor.merge") } - container = roaring.NewContainerBitmap(cell.N, d) + container = roaring.NewContainerBitmap(cell.BitN, d) case ContainerTypeRLE: d := toInterval16(cell.Data) container = roaring.NewContainerRun(d) diff --git a/rbf/cursor_test.go b/rbf/cursor_test.go index 9e98fafb3..bd8e67203 100644 --- a/rbf/cursor_test.go +++ b/rbf/cursor_test.go @@ -31,7 +31,7 @@ func TestCursor_FirstNext(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) @@ -95,7 +95,7 @@ func TestCursor_FirstNext_Quick(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() // Insert values in random order. if err := tx.CreateBitmap("x"); err != nil { @@ -153,7 +153,7 @@ func TestCursor_LastPrev(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) @@ -217,7 +217,7 @@ func TestCursor_LastPrev_Quick(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() // Insert values in random order. if err := tx.CreateBitmap("x"); err != nil { @@ -276,7 +276,7 @@ func TestCursor_Union(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) @@ -322,7 +322,7 @@ func TestCursor_Union(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() values := GenerateValues(rand, 10000) rows := ToRows(values) @@ -356,7 +356,7 @@ func TestCursor_Intersect(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() row := make([]uint64, rbf.ShardWidth/64) @@ -403,7 +403,7 @@ func TestCursor_Intersect(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() values := GenerateValues(rand, rand.Intn(10000)) rows := ToRows(values) @@ -447,7 +447,7 @@ func TestCursor_AddRoaring(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) @@ -466,7 +466,7 @@ func TestCursor_AddRoaring(t *testing.T) { return bm }(), wantChanged: false, - wantErr: true}, + wantErr: false}, { name: "initial Array", fieldview: "x", @@ -614,7 +614,7 @@ func TestCursor_RLETesting(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() //setup RLE if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) @@ -696,10 +696,10 @@ func TestCursor_RLETesting(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - changed, err := tx.Add("x", tt.args...) + changeCount, err := tx.Add("x", tt.args...) if tt.wantErr && err == nil { t.Errorf("No Error %v", err) - } else if tt.wantChanged && !changed { + } else if tt.wantChanged && changeCount == 0 { t.Errorf("No Change %v", err) } else if err != nil { t.Fatal(err) @@ -734,7 +734,7 @@ func TestCursor_RLEConversion(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() //setup RLE with full container if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) @@ -840,7 +840,7 @@ func TestCursor_UpdateBranchCells(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) } @@ -914,7 +914,7 @@ func TestCursor_SplitBranchCells(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) } @@ -964,7 +964,7 @@ func TestCursor_RemoveCells(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) } @@ -1006,7 +1006,7 @@ func TestCursor_PlayContainer(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) } @@ -1035,14 +1035,13 @@ func TestCursor_PlayContainer(t *testing.T) { if err := cur.First(); err != nil { panic(err) } - cur.Dump("fun.dot") } func TestCursor_OneBitmap(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) } @@ -1071,13 +1070,12 @@ func TestCursor_OneBitmap(t *testing.T) { if err := cur.First(); err != nil { panic(err) } - cur.Dump("fun.dot") } func TestCursor_GenerateAll(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) } @@ -1111,9 +1109,4 @@ func TestCursor_GenerateAll(t *testing.T) { if _, err := tx.AddRoaring("field/view/", bb); err != nil { panic(err) } - cur, err := tx.Cursor("field/view/") - if err != nil { - panic(err) - } - cur.Dump("fun.dot") } diff --git a/rbf/cursorx.go b/rbf/cursorx.go index 8a317c184..54febc5b0 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -93,10 +93,11 @@ func (c *Cursor) Dump(name string) { fmt.Fprintf(bufStdout, "\n}") bufStdout.Flush() } -func (c *Cursor) Row(rowID uint64) (*roaring.Bitmap, error) { + +func (c *Cursor) Row(shard, rowID uint64) (*roaring.Bitmap, error) { base := rowID * ShardWidth - offset := uint64(c.tx.db.Shard * ShardWidth) + offset := uint64(shard * ShardWidth) off := highbits(offset) hi0, hi1 := highbits(base), highbits((rowID+1)*ShardWidth) c.stack.index = 0 diff --git a/rbf/db.go b/rbf/db.go index 09f4b5aa9..9b642c1fb 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -54,20 +54,15 @@ type DB struct { // The maximum allowed database size. Required by mmap. MaxSize int64 - Shard int } // NewDB returns a new instance of DB. func NewDB(path string) *DB { - return NewDBWithShard(path, 0) -} -func NewDBWithShard(path string, shard int) *DB { return &DB{ txs: make(map[*Tx]struct{}), pageMap: immutable.NewMap(&uint32Hasher{}), Path: path, MaxSize: DefaultMaxSize, - Shard: shard, } } @@ -97,7 +92,7 @@ func (db *DB) Open() (err error) { db.mu.Lock() defer db.mu.Unlock() - if err := os.MkdirAll(filepath.Dir(db.Path), 0755); err != nil { + if err := os.MkdirAll(db.Path, 0755); err != nil { return err } else if db.file, err = os.OpenFile(db.DataPath(), os.O_WRONLY|os.O_CREATE, 0666); err != nil { return fmt.Errorf("open file: %w", err) @@ -569,7 +564,7 @@ func (db *DB) Begin(writable bool) (_ *Tx, err error) { // This page is only written at the end of a dirty transaction. page, err := db.readPage(db.pageMap, 0) if err != nil { - _ = tx.Rollback() + tx.Rollback() return nil, err } copy(tx.meta[:], page) @@ -613,7 +608,7 @@ func (db *DB) Check() error { if err != nil { return err } - defer func() { _ = tx.Rollback() }() + defer tx.Rollback() return tx.Check() } diff --git a/rbf/db_test.go b/rbf/db_test.go index f68b0f26e..ef4e75ade 100644 --- a/rbf/db_test.go +++ b/rbf/db_test.go @@ -119,14 +119,13 @@ func TestDB_Recovery(t *testing.T) { if err != nil { t.Fatal(err) } - defer MustRollback(t, tx) + defer tx.Rollback() if exists, err := tx.Contains("x", uint64(len(a))); exists || err != nil { t.Fatalf("Contains()=<%v,%#v>", exists, err) } else if exists, err := tx.Contains("x", uint64(len(a)-1)); !exists || err != nil { t.Fatalf("Contains()=<%v,%#v>", exists, err) - } else if err := tx.Rollback(); err != nil { - t.Fatal(err) } + tx.Rollback() }) } diff --git a/rbf/rbf.go b/rbf/rbf.go index 443b2f940..911532d15 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -21,8 +21,11 @@ import ( "errors" "fmt" "io" + "math" + "os" "unsafe" + "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/shardwidth" ) @@ -81,6 +84,8 @@ var ( ErrTxClosed = errors.New("transaction closed") ErrTxNotWritable = errors.New("transaction not writable") ErrBitmapNameRequired = errors.New("bitmap name required") + ErrBitmapNotFound = errors.New("bitmap not found") + ErrBitmapExists = errors.New("bitmap already exists") ) // Debug is just a temporary flag used for debugging. @@ -257,8 +262,15 @@ func align8(offset int) int { // leafCell represents a leaf cell. type leafCell struct { Key uint64 - Type int - N int + Type int // container type + + // N is the number of "things" in Data: + // for an array container the number of integers in the array. + // for an RLE, number of intervals. + // etc. + N int + + BitN int Data []byte } @@ -361,19 +373,70 @@ func (c *leafCell) firstValue() uint16 { } } +// lastValue the last value from the container. +func (c *leafCell) lastValue() uint16 { + switch c.Type { + case ContainerTypeArray: + a := toArray16(c.Data) + return a[len(a)-1] + case ContainerTypeRLE: + r := toInterval16(c.Data) + return r[len(r)-1].Last + case ContainerTypeBitmap: + a := toArray64(c.Data) + for i := len(a) - 1; i >= 0; i-- { + for j := 63; j >= 0; j-- { + if a[i]&(1< math.MaxUint16 { + return c.BitN + } + + switch c.Type { + case ContainerTypeArray: + return int(roaring.ArrayCountRange(toArray16(c.Data), start, end)) + case ContainerTypeRLE: + return int(roaring.RunCountRange(toInterval16(c.Data), start, end)) + case ContainerTypeBitmap: + return int(roaring.BitmapCountRange(toArray64(c.Data), start, end)) + default: + panic(fmt.Sprintf("invalid container type: %d", c.Type)) + } +} + func readLeafCellKey(page []byte, i int) uint64 { offset := readCellOffset(page, i) + assert(offset < len(page)) return *(*uint64)(unsafe.Pointer(&page[offset])) } func readLeafCell(page []byte, i int) leafCell { offset := readCellOffset(page, i) + + // cd ..; PILOSA_TXSRC=rbf go test -v -run TestFragment_TopN_IDs -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0" + // gives panic: runtime error: slice bounds out of range [16390:8192] here. buf := page[offset:] var cell leafCell cell.Key = *(*uint64)(unsafe.Pointer(&buf[0])) cell.Type = int(*(*uint32)(unsafe.Pointer(&buf[8]))) - cell.N = int(*(*uint32)(unsafe.Pointer(&buf[12]))) + cell.N = int(*(*uint16)(unsafe.Pointer(&buf[12]))) + cell.BitN = int(*(*uint16)(unsafe.Pointer(&buf[14]))) switch cell.Type { case ContainerTypeArray: @@ -410,7 +473,8 @@ func writeLeafCell(page []byte, i, offset int, cell leafCell) { writeCellOffset(page, i, offset) *(*uint64)(unsafe.Pointer(&page[offset])) = cell.Key *(*uint32)(unsafe.Pointer(&page[offset+8])) = uint32(cell.Type) - *(*uint32)(unsafe.Pointer(&page[offset+12])) = uint32(cell.N) + *(*uint16)(unsafe.Pointer(&page[offset+12])) = uint16(cell.N) + *(*uint16)(unsafe.Pointer(&page[offset+14])) = uint16(cell.BitN) assert(offset+16+len(cell.Data) <= PageSize) copy(page[offset+16:], cell.Data) } @@ -486,8 +550,11 @@ func search(n int, f func(int) int) (index int, exact bool) { return i, false } -/* -func pagedumpi(b []byte, indent string, writer io.Writer) { +func Pagedump(b []byte, indent string, writer io.Writer) { + if writer == nil { + writer = os.Stderr + } + pgno := readPageNo(b) if pgno == Magic32() { fmt.Fprintf(writer, "==META\n") @@ -501,6 +568,7 @@ func pagedumpi(b []byte, indent string, writer io.Writer) { // the page alone so this will output !PAGE for bitmap pages & invalid pages. switch { case flags&PageTypeLeaf != 0: + fmt.Fprintf(writer, "==LEAF pgno=%d flags=%d n=%d\n", pgno, flags, cellN) for i := 0; i < cellN; i++ { cell := readLeafCell(b, i) switch cell.Type { @@ -525,7 +593,6 @@ func pagedumpi(b []byte, indent string, writer io.Writer) { fmt.Fprintf(writer, "==!PAGE %d flags=%d\n", pgno, flags) } } -*/ func Walk(tx *Tx, pgno uint32, v func(uint32, []*RootRecord)) { for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; { @@ -563,3 +630,8 @@ func RowValues(b []uint64) []uint64 { } return a } + +// func caller(skip int) string { +// _, file, line, _ := runtime.Caller(skip + 1) +// return fmt.Sprintf("%s:%d", file, line) +// } diff --git a/rbf/rbf_test.go b/rbf/rbf_test.go index a576dbd5e..8b4768cd5 100644 --- a/rbf/rbf_test.go +++ b/rbf/rbf_test.go @@ -119,14 +119,6 @@ func MustBegin(tb testing.TB, db *rbf.DB, writable bool) *rbf.Tx { return tx } -// MustRollback rolls back a transaction or fails. -func MustRollback(tb testing.TB, tx *rbf.Tx) { - tb.Helper() - if err := tx.Rollback(); err != nil && err != rbf.ErrTxClosed { - tb.Logf("rollback error: %q", err) - } -} - // MustAddRandom adds values to a bitmap in a random order. func MustAddRandom(tb testing.TB, rand *rand.Rand, tx *rbf.Tx, name string, values ...uint64) { tb.Helper() diff --git a/rbf/tx.go b/rbf/tx.go index 8740697dc..d11c34909 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -14,9 +14,14 @@ package rbf import ( + "bytes" "fmt" "io" + "math" "sort" + "strconv" + "strings" + "sync" "github.com/benbjohnson/immutable" "github.com/pilosa/pilosa/v2/roaring" @@ -24,16 +29,34 @@ import ( // Tx represents a transaction. type Tx struct { + mu sync.RWMutex db *DB // parent db meta [PageSize]byte // copy of current meta page walID int64 // max WAL ID at start of tx pageMap *immutable.Map // mapping of database pages to WAL IDs writable bool // if true, tx can write dirty bool // if true, changes have been made + + // If Rollback() has already completed, don't do it again. + // Note db == nil means that commit has already been done. + rollbackDone bool + + // DeleteEmptyContainer lets us by default match the roaring + // behavior where an existing container has all its bits cleared + // but still sticks around in the database. + DeleteEmptyContainer bool +} + +// Writable returns true if the transaction can mutate data. +func (tx *Tx) Writable() bool { + return tx.writable } // Commit completes the transaction and persists data changes. func (tx *Tx) Commit() error { + tx.mu.Lock() + defer tx.mu.Unlock() + if tx.db == nil { return ErrTxClosed } @@ -57,39 +80,54 @@ func (tx *Tx) Commit() error { return tx.db.removeTx(tx) } -func (tx *Tx) Rollback() error { - if tx.db == nil { - return ErrTxClosed +func (tx *Tx) Rollback() { + tx.mu.Lock() + defer tx.mu.Unlock() + // allow Rollback to be called more than once. + if tx.rollbackDone { + return } + tx.rollbackDone = true + if tx.db == nil { + // Commit already done. + return + } + + // TODO(bbj): Invalidate DB if rollback fails. Possibly attempt reopen? // If any pages have been written, ensure we write a new meta page with // the rollback flag to mark the end of the transaction. This allows us to // discard pages in the transaction during playback of the WAL on open. if tx.dirty { if err := tx.writeMetaPage(MetaPageFlagRollback); err != nil { - return err + panic(err) } else if err := tx.db.SyncWAL(); err != nil { - return err + panic(err) } } - if err := tx.db.checkpoint(); err != nil { - return err - } + // turn on these error checks! we see + // panic: cannot find segment containing WAL page: 1 + // when running go test -v + // TestCursor_FirstNext_Quick/6 + // + //panicOn(tx.db.checkpoint()) + //panicOn(tx.db.removeTx(tx)) + + _ = tx.db.checkpoint() // Disconnect transaction from DB. - err := tx.db.removeTx(tx) - _ = err - /* - if err != nil { - //TODO need to fix this error - } - */ - return nil + _ = tx.db.removeTx(tx) } // Root returns the root page number for a bitmap. Returns 0 if the bitmap does not exist. func (tx *Tx) Root(name string) (uint32, error) { + tx.mu.RLock() + defer tx.mu.RUnlock() + return tx.root(name) +} + +func (tx *Tx) root(name string) (uint32, error) { records, err := tx.rootRecords() if err != nil { return 0, err @@ -97,14 +135,45 @@ func (tx *Tx) Root(name string) (uint32, error) { i := sort.Search(len(records), func(i int) bool { return records[i].Name >= name }) if i >= len(records) || records[i].Name != name { - return 0, fmt.Errorf("bitmap not found: %q", name) + return 0, ErrBitmapNotFound } return records[i].Pgno, nil } +// BitmapNames returns a list of all bitmap names. +func (tx *Tx) BitmapNames() ([]string, error) { + tx.mu.RLock() + defer tx.mu.RUnlock() + + if tx.db == nil { + return nil, ErrTxClosed + } + + // Read list of root records. + records, err := tx.rootRecords() + if err != nil { + return nil, err + } + + // Convert to a list of strings. + names := make([]string, len(records)) + for i := range records { + names[i] = records[i].Name + } + return names, nil +} + // CreateBitmap creates a new empty bitmap with the given name. // Returns an error if the bitmap already exists. func (tx *Tx) CreateBitmap(name string) error { + tx.mu.Lock() + defer tx.mu.Unlock() + return tx.createBitmap(name) +} + +func (tx *Tx) createBitmap(name string) error { + //vv("createBitmap(name='%v'", name) + if tx.db == nil { return ErrTxClosed } else if !tx.writable { @@ -122,7 +191,7 @@ func (tx *Tx) CreateBitmap(name string) error { // Find btree by name. Exit if already exists. index := sort.Search(len(records), func(i int) bool { return records[i].Name >= name }) if index < len(records) && records[index].Name == name { - return fmt.Errorf("bitmap already exists: %q", name) + return ErrBitmapExists } //fmt.Println("CREATE BITMAP", name, index) @@ -153,6 +222,22 @@ func (tx *Tx) CreateBitmap(name string) error { return nil } +// CreateBitmapIfNotExists creates a new empty bitmap with the given name. +// This is a no-op if the bitmap already exists. +func (tx *Tx) CreateBitmapIfNotExists(name string) error { + if err := tx.CreateBitmap(name); err != nil && err != ErrBitmapExists { + return err + } + return nil +} + +func (tx *Tx) createBitmapIfNotExists(name string) error { + if err := tx.createBitmap(name); err != nil && err != ErrBitmapExists { + return err + } + return nil +} + /* func dump(r []*RootRecord) { for _, i := range r { @@ -165,6 +250,9 @@ func dump(r []*RootRecord) { // DeleteBitmap removes a bitmap with the given name. // Returns an error if the bitmap does not exist. func (tx *Tx) DeleteBitmap(name string) error { + tx.mu.Lock() + defer tx.mu.Unlock() + if tx.db == nil { return ErrTxClosed } else if !tx.writable { @@ -200,9 +288,55 @@ func (tx *Tx) DeleteBitmap(name string) error { return nil } +// DeleteBitmapsWithPrefix removes all bitmaps with a given prefix. +func (tx *Tx) DeleteBitmapsWithPrefix(prefix string) error { + tx.mu.Lock() + defer tx.mu.Unlock() + + if tx.db == nil { + return ErrTxClosed + } else if !tx.writable { + return ErrTxNotWritable + } + + // Read list of root records. + records, err := tx.rootRecords() + if err != nil { + return err + } + + for i := 0; i < len(records); i++ { + record := records[i] + + // Skip bitmaps without matching prefix. + if !strings.HasPrefix(record.Name, prefix) { + continue + } + + // Deallocate all pages in the tree. + if err := tx.deallocateTree(record.Pgno); err != nil { + return err + } + + // Delete from record list. + records = append(records[:i], records[i+1:]...) + i-- + } + + // Rewrite record pages. + if err := tx.writeRootRecordPages(records); err != nil { + return fmt.Errorf("write bitmaps: %w", err) + } + + return nil +} + // RenameBitmap updates the name of an existing bitmap. // Returns an error if the bitmap does not exist. func (tx *Tx) RenameBitmap(oldname, newname string) error { + tx.mu.Lock() + defer tx.mu.Unlock() + if tx.db == nil { return ErrTxClosed } else if !tx.writable { @@ -314,78 +448,105 @@ func (tx *Tx) writeRootRecordPages(records []*RootRecord) (err error) { } // Add sets a given bit on the bitmap. -func (tx *Tx) Add(name string, a ...uint64) (changed bool, err error) { +func (tx *Tx) Add(name string, a ...uint64) (changeCount int, err error) { + //vv("rbf Tx.Add(a='%#v')", a) + + tx.mu.Lock() + defer tx.mu.Unlock() + if tx.db == nil { - return false, ErrTxClosed + return 0, ErrTxClosed } else if !tx.writable { - return false, ErrTxNotWritable + return 0, ErrTxNotWritable } else if name == "" { - return false, ErrBitmapNameRequired + return 0, ErrBitmapNameRequired } - c, err := tx.Cursor(name) + if err := tx.createBitmapIfNotExists(name); err != nil { + return 0, err + } + + c, err := tx.cursor(name) if err != nil { - return false, err + return 0, err } for _, v := range a { if vchanged, err := c.Add(v); err != nil { - return changed, err + return changeCount, err } else if vchanged { - changed = true + changeCount++ } } - return changed, nil + return changeCount, nil } // Remove unsets a given bit on the bitmap. -func (tx *Tx) Remove(name string, a ...uint64) (changed bool, err error) { +func (tx *Tx) Remove(name string, a ...uint64) (changeCount int, err error) { + tx.mu.Lock() + defer tx.mu.Unlock() + if tx.db == nil { - return false, ErrTxClosed + return 0, ErrTxClosed } else if !tx.writable { - return false, ErrTxNotWritable + return 0, ErrTxNotWritable } else if name == "" { - return false, ErrBitmapNameRequired + return 0, ErrBitmapNameRequired } - c, err := tx.Cursor(name) + c, err := tx.cursor(name) if err != nil { - return false, err + return 0, err + } else if c == nil { + return 0, nil } for _, v := range a { if vchanged, err := c.Remove(v); err != nil { - return changed, err + return changeCount, err } else if vchanged { - changed = true + changeCount++ } } - return changed, nil + return changeCount, nil } // Contains returns true if the given bit is set on the bitmap. func (tx *Tx) Contains(name string, v uint64) (bool, error) { + tx.mu.RLock() + defer tx.mu.RUnlock() + if tx.db == nil { return false, ErrTxClosed } else if name == "" { return false, ErrBitmapNameRequired } - c, err := tx.Cursor(name) + c, err := tx.cursor(name) if err != nil { return false, err + } else if c == nil { + return false, nil } return c.Contains(v) } // Cursor returns an instance of a cursor this bitmap. func (tx *Tx) Cursor(name string) (*Cursor, error) { + tx.mu.RLock() + defer tx.mu.RUnlock() + return tx.cursor(name) +} + +func (tx *Tx) cursor(name string) (*Cursor, error) { if tx.db == nil { return nil, ErrTxClosed } else if name == "" { return nil, ErrBitmapNameRequired } - root, err := tx.Root(name) - if err != nil { + root, err := tx.root(name) + if err == ErrBitmapNotFound { + return nil, nil + } else if err != nil { return nil, err } @@ -394,8 +555,109 @@ func (tx *Tx) Cursor(name string) (*Cursor, error) { return &c, nil } +// RoaringBitmap returns a bitmap as a Roaring bitmap. +func (tx *Tx) RoaringBitmap(name string) (*roaring.Bitmap, error) { + tx.mu.RLock() + defer tx.mu.RUnlock() + + if tx.db == nil { + return nil, ErrTxClosed + } else if name == "" { + return nil, ErrBitmapNameRequired + } + + c, err := tx.cursor(name) + if err != nil { + return nil, err + } else if c == nil { + return roaring.NewSliceBitmap(), nil + } + + other := roaring.NewSliceBitmap() + if err := c.First(); err == io.EOF { + return other, nil + } else if err != nil { + return nil, err + } + + for { + if err := c.Next(); err == io.EOF { + return other, nil + } else if err != nil { + return nil, err + } + + cell := c.cell() + other.Containers.Put(cell.Key, toContainer(cell, tx)) + } +} + +// Container returns a Roaring container by key. +func (tx *Tx) Container(name string, key uint64) (*roaring.Container, error) { + tx.mu.RLock() + defer tx.mu.RUnlock() + + if tx.db == nil { + return nil, ErrTxClosed + } else if name == "" { + return nil, ErrBitmapNameRequired + } + + c, err := tx.cursor(name) + if err != nil { + return nil, err + } else if c == nil { + return nil, err + } else if exact, err := c.Seek(key); err != nil || !exact { + return nil, err + } + return toContainer(c.cell(), tx), nil +} + +// PutContainer inserts a container into a bitmap. Overwrites if key already exists. +func (tx *Tx) PutContainer(name string, key uint64, ct *roaring.Container) error { + tx.mu.Lock() + defer tx.mu.Unlock() + + if ct.N() == 0 { + return nil + } + cell := ConvertToLeafArgs(key, ct) + + if err := tx.createBitmapIfNotExists(name); err != nil { + return err + } + + c, err := tx.cursor(name) + if err != nil { + return err + } else if _, err := c.Seek(cell.Key); err != nil { + return err + } + return c.putLeafCell(cell) +} + +// RemoveContainer removes a container from the bitmap by key. +func (tx *Tx) RemoveContainer(name string, key uint64) error { + tx.mu.Lock() + defer tx.mu.Unlock() + + c, err := tx.cursor(name) + if err != nil { + return err + } else if c == nil { + return nil + } else if exact, err := c.Seek(key); err != nil || !exact { + return err + } + return c.deleteLeafCell(key) +} + // Check verifies the integrity of the database. func (tx *Tx) Check() error { + tx.mu.RLock() + defer tx.mu.RUnlock() + if tx.db == nil { return ErrTxClosed } @@ -677,12 +939,20 @@ func (tx *Tx) writeMetaPage(flag uint32) error { } func (tx *Tx) AddRoaring(name string, bm *roaring.Bitmap) (changed bool, err error) { - c, err := tx.Cursor(name) + tx.mu.RLock() + defer tx.mu.RUnlock() + + if err := tx.createBitmapIfNotExists(name); err != nil { + return false, err + } + + c, err := tx.cursor(name) if err != nil { return false, err } return c.AddRoaring(bm) } + func (tx *Tx) leafCellBitmap(pgno uint32) (uint32, []uint64, error) { page, err := tx.readPage(pgno) if err != nil { @@ -690,3 +960,611 @@ func (tx *Tx) leafCellBitmap(pgno uint32) (uint32, []uint64, error) { } return pgno, toArray64(page), err } + +func (tx *Tx) ContainerIterator(name string, key uint64) (citer roaring.ContainerIterator, found bool, err error) { + tx.mu.RLock() + defer tx.mu.RUnlock() + + var c *Cursor + c, err = tx.cursor(name) + if c == nil && err == nil { + // nothing available. + citer = &emptyContainerIterator{} + return + } + if err != nil { + return + } + + // INVAR: c is not nil + + err = c.First() + if err != nil { + return + } + ci := &containerIterator{cursor: c} + citer = ci + + return citer, true, nil +} + +func (tx *Tx) ForEach(name string, fn func(i uint64) error) error { + return tx.ForEachRange(name, 0, math.MaxUint64, fn) +} + +func (tx *Tx) ForEachRange(name string, start, end uint64, fn func(uint64) error) error { + tx.mu.RLock() + defer tx.mu.RUnlock() + + c, err := tx.cursor(name) + if err != nil { + return err + } else if c == nil { + return nil + } else if _, err := c.Seek(highbits(start)); err != nil { + return err + } + + for { + if err := c.Next(); err == io.EOF { + return nil + } else if err != nil { + return err + } + + switch cell := c.cell(); cell.Type { + case ContainerTypeArray: + for _, lo := range toArray16(cell.Data) { + v := cell.Key<<16 | uint64(lo) + if v < start { + continue + } else if v > end { + return nil + } else if err := fn(v); err != nil { + return err + } + } + case ContainerTypeRLE: + for _, r := range toInterval16(cell.Data) { + for lo := int(r.Start); lo <= int(r.Last); lo++ { + v := cell.Key<<16 | uint64(lo) + if v < start { + continue + } else if v > end { + return nil + } else if err := fn(v); err != nil { + return err + } + } + } + case ContainerTypeBitmap: + for i, bits := range toArray64(cell.Data) { + for j := uint(0); j < 64; j++ { + if bits&(1< end { + return nil + } else if err := fn(v); err != nil { + return err + } + } + } + default: + panic(fmt.Sprintf("invalid container type: %d", cell.Type)) + } + } +} + +func (tx *Tx) Count(name string) (uint64, error) { + tx.mu.RLock() + defer tx.mu.RUnlock() + + c, err := tx.cursor(name) + if err != nil { + return 0, err + } else if c == nil { + return 0, nil + } else if err := c.First(); err != nil { + return 0, err + } + + var n uint64 + for { + if err := c.Next(); err == io.EOF { + break + } else if err != nil { + return 0, err + } + + n += uint64(c.cell().BitN) + } + return n, nil +} + +func (tx *Tx) Max(name string) (uint64, error) { + tx.mu.RLock() + defer tx.mu.RUnlock() + + c, err := tx.cursor(name) + if err != nil { + return 0, err + } else if c == nil { + return 0, nil + } else if err := c.Last(); err == io.EOF { + return 0, nil + } else if err != nil { + return 0, err + } + + cell := c.cell() + return uint64((cell.Key << 16) | uint64(cell.lastValue())), nil +} + +func (tx *Tx) Min(name string) (uint64, bool, error) { + tx.mu.RLock() + defer tx.mu.RUnlock() + + c, err := tx.cursor(name) + if err != nil { + return 0, false, err + } else if c == nil { + return 0, false, nil + } else if err := c.First(); err == io.EOF { + return 0, false, nil + } else if err != nil { + return 0, false, err + } + + cell := c.cell() + return uint64((cell.Key << 16) | uint64(cell.firstValue())), true, nil +} + +func (tx *Tx) UnionInPlace(name string, others ...*roaring.Bitmap) error { + panic("TODO") +} + +// roaring.countRange counts the number of bits set between [start, end). +func (tx *Tx) CountRange(name string, start, end uint64) (uint64, error) { + tx.mu.RLock() + defer tx.mu.RUnlock() + + if start >= end { + return 0, nil + } + + skey := highbits(start) + ekey := highbits(end) + + csr, err := tx.cursor(name) + if err != nil { + return 0, err + } else if csr == nil { + return 0, nil + } + + exact, err := csr.Seek(skey) + _ = exact + if err == io.EOF { + return 0, nil + } else if err != nil { + return 0, err + } + + var n uint64 + for { + if err := csr.Next(); err == io.EOF { + break + } else if err != nil { + return 0, err + } + + c := csr.cell() + k := c.Key + if k > ekey { + break + } + + // If range is entirely in one container then just count that range. + if skey == ekey { + return uint64(c.countRange(int32(lowbits(start)), int32(lowbits(end)))), nil + } + // INVAR: skey < ekey + + // k > ekey handles the case when start > end and where start and end + // are in different containers. Same container case is already handled above. + if k > ekey { + break + } + if k == skey { + n += uint64(c.countRange(int32(lowbits(start)), roaring.MaxContainerVal+1)) + continue + } + if k < ekey { + n += uint64(c.BitN) + continue + } + if k == ekey { + n += uint64(c.countRange(0, int32(lowbits(end)))) + break + } + } + return n, nil +} + +func (tx *Tx) OffsetRange(name string, offset, start, endx uint64) (*roaring.Bitmap, error) { + if lowbits(offset) != 0 { + panic("offset must not contain low bits") + } else if lowbits(start) != 0 { + panic("range start must not contain low bits") + } else if lowbits(endx) != 0 { + panic("range endx must not contain low bits") + } + + tx.mu.RLock() + defer tx.mu.RUnlock() + + c, err := tx.cursor(name) + if err != nil { + return nil, err + } + + other := roaring.NewSliceBitmap() + off := highbits(offset) + hi0, hi1 := highbits(start), highbits(endx) + + if c == nil { + // bitmap not found. Match what roaring does and return nil in this case. + return other, nil + } + + if _, err := c.Seek(hi0); err == io.EOF { + return other, nil + } else if err != nil { + return nil, err + } + + for { + if err := c.Next(); err == io.EOF { + break + } else if err != nil { + return nil, err + } + + cell := c.cell() + ckey := cell.Key + + // >= hi1 is correct b/c endx cannot have any lowbits set. + if ckey >= hi1 { + break + } + other.Containers.Put(off+(ckey-hi0), toContainer(cell, tx)) + } + return other, nil +} + +// containerIterator wraps Cursor to implement roaring.ContainerIterator. +type containerIterator struct { + cursor *Cursor +} + +// Close is a no-op. It exists to implement the roaring.ContainerIterator interface. +func (itr *containerIterator) Close() {} + +// Next moves the iterator to the next container. +func (itr *containerIterator) Next() bool { + err := itr.cursor.Next() + return err == nil +} + +// Value returns the current key & container. +func (itr *containerIterator) Value() (uint64, *roaring.Container) { + cell := itr.cursor.cell() + return cell.Key, toContainer(cell, itr.cursor.tx) +} + +// always returns false for Next() +type emptyContainerIterator struct{} + +func (si *emptyContainerIterator) Close() {} + +func (si *emptyContainerIterator) Next() bool { + return false +} +func (si *emptyContainerIterator) Value() (uint64, *roaring.Container) { + panic("emptyContainerIterator never has any Values") +} + +func (tx *Tx) Dump(index string) { + fmt.Println(tx.DumpString(index)) +} +func (tx *Tx) DumpString(index string) (r string) { + + r = "allkeys:[\n" + + // grab root records, for a list of bitmaps. + records, err := tx.rootRecords() + panicOn(err) + n := 0 + for _, rr := range records { + c, err := tx.cursor(rr.Name) + panicOn(err) + err = c.First() // First will rewind to beginning. + if err == io.EOF { + r += "" + n++ + continue + } + panicOn(err) + for { + err := c.Next() + if err == io.EOF { + break + } + panicOn(err) + cell := c.cell() + ckey := cell.Key + ct := toContainer(cell, tx) + + s := stringOfCkeyCt(ckey, ct, rr.Name, index) + r += s + n++ + } + } + if n == 0 { + return "" + } + // note that we can have a bitmap present, but it can be empty + r += "]\n all-in-blake3:" + blake3sum16([]byte(r)) + "\n" + + return "rbf-" + r +} + +func containerToBytes(ct *roaring.Container) []byte { + ty := roaring.ContainerType(ct) + switch ty { + case containerNil: + panic("nil container") + case containerArray: + return fromArray16(roaring.AsArray(ct)) + case containerBitmap: + return fromArray64(roaring.AsBitmap(ct)) + case containerRun: + return fromInterval16(roaring.AsRuns(ct)) + } + panic(fmt.Sprintf("unknown container type '%v'", int(ty))) +} + +func badgerKey(index, field, view string, shard uint64, roaringContainerKey uint64) []byte { + // The %020d which adds zero padding up to 20 runes is required to + // allow the textual sort to accurately + // reflect a numeric sort order. This is because, as a string, + // math.MaxUint64 is 20 bytes long. + // Example of such a badgerKey with a container-key that is math.MaxUint64: + // ...........................................12345678901234567890 + // idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615 + + prefix := badgerPrefix(index, field, view, shard) + ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey)) + bkey := append(prefix, ckey...) + MustValidateKey(bkey) + return bkey +} + +// badgerPrefix returns everything from badgerKey up to and +// including the '@' fune in a badger key. The prefix excludes the roaring container key itself. +// NB must be kept in sync with badgerKey() and badgerKeyExtractContainerKey(). +func badgerPrefix(index, field, view string, shard uint64) []byte { + return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@", index, field, view, shard)) +} + +// MustValidatekey will panic on a bad badgerKey with an informative message. +func MustValidateKey(bkey []byte) { + n := len(bkey) + if n < 56 { + panic(fmt.Sprintf("bkey too short min size is 56 but we see %v in '%v'", n, string(bkey))) + } + beforeCkey := bkey[n-26 : n-20] + if !bytes.Equal(beforeCkey, ckeyPartExpected) { + panic(fmt.Sprintf(`bkey did not have expected ";ckey@" at 26 bytes from the end of the bkey '%v'; instead had '%v'`, string(bkey), string(beforeCkey))) + } +} + +func bitmapAsString(rbm *roaring.Bitmap) (r string) { + r = "c(" + slc := rbm.Slice() + width := 0 + s := "" + for _, v := range slc { + if width == 0 { + s = fmt.Sprintf("%v", v) + } else { + s = fmt.Sprintf(", %v", v) + } + width += len(s) + r += s + if width > 70 { + r += ",\n" + width = 0 + } + } + if width == 0 && len(r) > 2 { + r = r[:len(r)-2] + } + return r + ")" +} + +// should really be exported from the pilosa/roaring package so we don't get out of sync... +const ( + containerNil byte = iota // no container + containerArray // slice of bit position values + containerBitmap // slice of 1024 uint64s + containerRun // container of run-encoded bits +) + +var ckeyPartExpected = []byte(";ckey@") + +func invName(rbfName string) (field, view string, shard uint64) { + s := strings.Split(rbfName, "\x00") + if len(s) != 3 { + panic("should have 3 parts") + } + field = s[0] + view = s[1] + var err error + shard, err = strconv.ParseUint(s[2], 10, 64) + panicOn(err) + return +} + +func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName, index string) (s string) { + + by := containerToBytes(ct) + hash := blake3sum16(by) + + cts := roaring.NewSliceContainers() + cts.Put(ckey, ct) + rbm := &roaring.Bitmap{Containers: cts} + srbm := bitmapAsString(rbm) + + field, view, shard := invName(rrName) + bkey := string(badgerKey(index, field, view, shard, ckey)) + + s = fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, ct.N()) + s += " ......." + srbm + "\n" + return +} + +func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { + + // begin write boilerplate + if tx.db == nil { + err = ErrTxClosed + return + } else if !tx.writable { + err = ErrTxNotWritable + return + } else if name == "" { + err = ErrBitmapNameRequired + return + } + + if err = tx.createBitmapIfNotExists(name); err != nil { + return + } + // end write boilerplate + + n := itr.Len() + if n == 0 { + return + } + rowSet = make(map[uint64]int) + + var currRow uint64 + + var oldC *roaring.Container + for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { + if rowSize != 0 { + currRow = itrKey / rowSize + } + nsynth := int(synthC.N()) + if nsynth == 0 { + continue + } + // INVAR: nsynth > 0 + + oldC, err = tx.Container(name, itrKey) + panicOn(err) + if err != nil { + return + } + + if oldC == nil || oldC.N() == 0 { + // no container at the itrKey in badger (or all zero container). + if clear { + // changed of 0 and empty rowSet is perfect, no need to change the defaults. + continue + } else { + + changed += nsynth + rowSet[currRow] += nsynth + + err = tx.PutContainer(name, itrKey, synthC) + if err != nil { + return + } + continue + } + } + + if clear { + existN := oldC.N() // number of bits set in the old container + newC := oldC.Difference(synthC) + + // update rowSet and changes + if newC.N() == existN { + // INVAR: do changed need adjusting? nope. same bit count, + // so no change could have happened. + continue + } else { + changes := int(existN - newC.N()) + changed += changes + rowSet[currRow] -= changes + + if tx.DeleteEmptyContainer && newC.N() == 0 { + err = tx.RemoveContainer(name, itrKey) + if err != nil { + return + } + continue + } + err = tx.PutContainer(name, itrKey, newC) + if err != nil { + return + } + continue + } + } else { + // setting bits + + existN := oldC.N() + if existN == roaring.MaxContainerVal+1 { + // completely full container already, set will do nothing. so changed of 0 default is perfect. + continue + } + if existN == 0 { + // can nsynth be zero? No, because of the continue/invariant above where nsynth > 0 + changed += nsynth + rowSet[currRow] += nsynth + err = tx.PutContainer(name, itrKey, synthC) + if err != nil { + return + } + continue + } + + newC := oldC.UnionInPlace(synthC) + + if roaring.ContainerType(newC) == containerBitmap { + newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it. + } + if newC.N() != existN { + changes := int(newC.N() - existN) + changed += changes + rowSet[currRow] += changes + + err = tx.PutContainer(name, itrKey, newC) + if err != nil { + panicOn(err) + return + } + continue + } + } + } + return +} diff --git a/rbf/tx_test.go b/rbf/tx_test.go index 2af3f4bfc..e245b73da 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -29,13 +29,13 @@ func TestTx_CommitRollback(t *testing.T) { defer MustCloseDB(t, db) // Create bitmap in transaction but rollback. - if tx, err := db.Begin(true); err != nil { + tx, err := db.Begin(true) + if err != nil { t.Fatal(err) } else if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) - } else if err := tx.Rollback(); err != nil { - t.Fatal(err) } + tx.Rollback() // Create bitmap in transaction again but commit. if tx, err := db.Begin(true); err != nil { @@ -49,8 +49,8 @@ func TestTx_CommitRollback(t *testing.T) { // Create bitmap again but it should fail as it already exists. if tx, err := db.Begin(true); err != nil { t.Fatal(err) - } else if err := tx.CreateBitmap("x"); err == nil || err.Error() != `bitmap already exists: "x"` { - _ = tx.Rollback() + } else if err := tx.CreateBitmap("x"); err == nil || err != rbf.ErrBitmapExists { + tx.Rollback() t.Fatal(err) } else if err := tx.Commit(); err != nil { t.Fatal(err) @@ -62,13 +62,13 @@ func TestTx_CommitRollback(t *testing.T) { defer func() { MustCloseDB(t, db) }() // Create bitmap in transaction but rollback. - if tx, err := db.Begin(true); err != nil { + tx, err := db.Begin(true) + if err != nil { t.Fatal(err) } else if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) - } else if err := tx.Rollback(); err != nil { - t.Fatal(err) } + tx.Rollback() db = MustReopenDB(t, db) // Create bitmap in transaction again but commit. @@ -84,8 +84,8 @@ func TestTx_CommitRollback(t *testing.T) { // Create bitmap again but it should fail as it already exists. if tx, err := db.Begin(true); err != nil { t.Fatal(err) - } else if err := tx.CreateBitmap("x"); err == nil || err.Error() != `bitmap already exists: "x"` { - _ = tx.Rollback() + } else if err := tx.CreateBitmap("x"); err == nil || err != rbf.ErrBitmapExists { + tx.Rollback() t.Fatal(err) } else if err := tx.Commit(); err != nil { t.Fatal(err) @@ -102,7 +102,7 @@ func TestTx_CommitRollback(t *testing.T) { tx0 := MustBegin(t, db, true) go func() { <-ch0 - _ = tx0.Rollback() + tx0.Rollback() }() // Start separate write transaction in different goroutine. @@ -134,7 +134,7 @@ func TestTx_Add(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) @@ -167,7 +167,7 @@ func TestTx_DeleteBitmap(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() // Create bitmap & add value. if err := tx.CreateBitmap("x"); err != nil { @@ -192,7 +192,7 @@ func TestTx_RenameBitmap(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() // Create bitmap & add value. if err := tx.CreateBitmap("x"); err != nil { @@ -226,7 +226,7 @@ func TestTx_Add_Quick(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() values := GenerateValues(rand, 10000) if err := tx.CreateBitmap("x"); err != nil { @@ -265,7 +265,7 @@ func TestTx_AddRemove_Quick(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() values := GenerateValues(rand, 10000) if err := tx.CreateBitmap("x"); err != nil { @@ -314,7 +314,7 @@ func TestTx_Multiple_CreateBitmap(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() values := GenerateValues(rand, 2) if err := tx.CreateBitmap("x/1"); err != nil { @@ -332,7 +332,7 @@ func TestTx_Multiple_CreateBitmap(t *testing.T) { } tx1 := MustBegin(t, db, true) - defer func() { _ = tx1.Rollback() }() + defer tx1.Rollback() if err := tx1.CreateBitmap("x/2"); err != nil { t.Fatal(err) @@ -353,7 +353,7 @@ func TestTx_CursorCrashArray(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) } @@ -379,7 +379,7 @@ func TestTx_CursorCrashBitmap(t *testing.T) { db := MustOpenDB(t) defer MustCloseDB(t, db) tx := MustBegin(t, db, true) - defer MustRollback(t, tx) + defer tx.Rollback() if err := tx.CreateBitmap("x"); err != nil { t.Fatal(err) } @@ -418,7 +418,7 @@ func BenchmarkTx_Add(b *testing.B) { db := MustOpenDB(b) defer MustCloseDB(b, db) tx := MustBegin(b, db, true) - defer MustRollback(b, tx) + defer tx.Rollback() for _, v := range values { if _, err := tx.Add("x", v); err != nil { @@ -446,7 +446,7 @@ func BenchmarkTx_Contains(b *testing.B) { db := MustOpenDB(b) defer MustCloseDB(b, db) tx := MustBegin(b, db, true) - defer MustRollback(b, tx) + defer tx.Rollback() b.ResetTimer() t := time.Now() @@ -464,3 +464,29 @@ func BenchmarkTx_Contains(b *testing.B) { }) } } + +func TestTx_Dump(t *testing.T) { + db := MustOpenDB(t) + defer MustCloseDB(t, db) + tx := MustBegin(t, db, true) + defer tx.Rollback() + + index, field, view, shard := "i", "f", "v", uint64(15) + nm := rbfName(field, view, shard) + + if err := tx.CreateBitmap(nm); err != nil { + t.Fatal(err) + } else if _, err := tx.Add(nm, 0x00000001, 0x00000002, 0x00010003, 0x00030004); err != nil { + t.Fatal(err) + } + + // test that we don't crash, and get *something* back + s := tx.DumpString(index) + if s == "" { + panic("should have had 3 containers!") + } +} + +func rbfName(field, view string, shard uint64) string { + return fmt.Sprintf("%s\x00%s\x00%d", field, view, shard) +} diff --git a/rbf/vprint.go b/rbf/vprint.go new file mode 100644 index 000000000..dfd630c70 --- /dev/null +++ b/rbf/vprint.go @@ -0,0 +1,169 @@ +// home: https://github.com/glyerine/vprint +// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved. +// License: MIT +// +// MIT License +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package rbf + +import ( + "fmt" + "io" + "os" + "path" + "runtime" + "runtime/debug" + "sync" + "time" +) + +const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00" +const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00" + +// for tons of debug output +var VerboseVerbose bool = false + +// convience functions for . import +var pp = PP +var vv = VV + +var panicOn = PanicOn + +func init() { + // keeper linter happy + _ = pp + _ = vv +} + +func PanicOn(err error) { + if err != nil { + panic(err) + } +} + +func PP(format string, a ...interface{}) { + if VerboseVerbose { + TSPrintf(format, a...) + } +} + +func VV(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +func AlwaysPrintf(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +var tsPrintfMut sync.Mutex + +// time-stamped printf +func TSPrintf(format string, a ...interface{}) { + tsPrintfMut.Lock() + Printf("\n%s %s ", FileLine(3), ts()) + Printf(format+"\n", a...) + tsPrintfMut.Unlock() +} + +// get timestamp for logging purposes +func ts() string { + return time.Now().Format(RFC3339UsecTz0) +} + +// so we can multi write easily, use our own printf +var OurStdout io.Writer = os.Stdout + +// Printf formats according to a format specifier and writes to standard output. +// It returns the number of bytes written and any write error encountered. +func Printf(format string, a ...interface{}) (n int, err error) { + return fmt.Fprintf(OurStdout, format, a...) +} + +func FileLine(depth int) string { + _, fileName, fileLine, ok := runtime.Caller(depth) + var s string + if ok { + s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine) + } else { + s = "" + } + return s +} + +func stack() string { + return string(debug.Stack()) +} + +func FileExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return false + } + return true +} + +func DirExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return true + } + return false +} + +func FileSize(name string) (int64, error) { + fi, err := os.Stat(name) + if err != nil { + return -1, err + } + return fi.Size(), nil +} + +// Caller returns the name of the calling function. +func Caller(upStack int) string { + // elide ourself and runtime.Callers + target := upStack + 2 + + pc := make([]uintptr, target+2) + n := runtime.Callers(0, pc) + + f := runtime.Frame{Function: "unknown"} + if n > 0 { + frames := runtime.CallersFrames(pc[:n]) + for i := 0; i <= target; i++ { + contender, more := frames.Next() + if i == target { + f = contender + } + if !more { + break + } + } + } + return f.Function +} + +var _ = stack // happy linter diff --git a/roaring/container_stash.go b/roaring/container_stash.go index 3531a0317..095f462e5 100644 --- a/roaring/container_stash.go +++ b/roaring/container_stash.go @@ -387,7 +387,7 @@ func (c *Container) bitmap() []uint64 { // is provided. The target should be zeroed, or this becomes an implicit // union. func (c *Container) AsBitmap(target []uint64) (out []uint64) { - if c.typeID == containerBitmap { + if c != nil && c.typeID == containerBitmap { return c.bitmap() } // Reminder: len(nil) == 0. @@ -399,6 +399,10 @@ func (c *Container) AsBitmap(target []uint64) (out []uint64) { out[i] = 0 } } + // A nil *Container is a valid empty container. + if c == nil { + return out + } if c.typeID == containerArray { a := c.array() for _, v := range a { diff --git a/roaring/roaring.go b/roaring/roaring.go index 7f755f995..9219b01ef 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2860,20 +2860,19 @@ func (c *Container) countRange(start, end int32) (n int32) { return 0 } if c.isArray() { - return c.arrayCountRange(start, end) + return ArrayCountRange(c.array(), start, end) } else if c.isRun() { - return c.runCountRange(start, end) + return RunCountRange(c.runs(), start, end) } - return c.bitmapCountRange(start, end) + return BitmapCountRange(c.bitmap(), start, end) } -func (c *Container) arrayCountRange(start, end int32) (n int32) { +func ArrayCountRange(array []uint16, start, end int32) (n int32) { if roaringParanoia { if start > end { panic(fmt.Sprintf("counting in range but %v > %v", start, end)) } } - array := c.array() i := int32(sort.Search(len(array), func(i int) bool { return int32(array[i]) >= start })) for ; i < int32(len(array)); i++ { v := int32(array[i]) @@ -2885,7 +2884,7 @@ func (c *Container) arrayCountRange(start, end int32) (n int32) { return n } -func (c *Container) bitmapCountRange(start, end int32) int32 { +func BitmapCountRange(bitmap []uint64, start, end int32) int32 { if roaringParanoia { if start > end { panic(fmt.Sprintf("counting in range but %v > %v", start, end)) @@ -2894,7 +2893,6 @@ func (c *Container) bitmapCountRange(start, end int32) int32 { var n uint64 i, j := start/64, end/64 // Special case when start and end fall in the same word. - bitmap := c.bitmap() if i == j { offi, offj := uint(start%64), uint(64-end%64) n += popcount((bitmap[i] >> offi) << (offj + offi)) @@ -2921,13 +2919,13 @@ func (c *Container) bitmapCountRange(start, end int32) int32 { return int32(n) } -func (c *Container) runCountRange(start, end int32) (n int32) { +// RunCountRange returns the ranged bit count for RLE pairs. +func RunCountRange(runs []Interval16, start, end int32) (n int32) { if roaringParanoia { if start > end { panic(fmt.Sprintf("counting in range but %v > %v", start, end)) } } - runs := c.runs() for _, iv := range runs { // iv is before range if int32(iv.Last) < start { @@ -3837,12 +3835,12 @@ func (c *Container) check() error { a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(array), c.N())) } } else if c.isRun() { - n := c.runCountRange(0, MaxContainerVal+1) + n := RunCountRange(c.runs(), 0, MaxContainerVal+1) if n != c.N() { a.Append(fmt.Errorf("run count mismatch: count=%d, n=%d", n, c.N())) } } else if c.isBitmap() { - if n := c.bitmapCountRange(0, MaxContainerVal+1); n != c.N() { + if n := BitmapCountRange(c.bitmap(), 0, MaxContainerVal+1); n != c.N() { a.Append(fmt.Errorf("bitmap count mismatch: count=%d, n=%d", n, c.N())) } } else { @@ -4052,7 +4050,7 @@ func intersectionCountRunRun(a, b *Container) (n int32) { func intersectionCountBitmapRun(a, b *Container) (n int32) { statsHit("intersectionCount/BitmapRun") for _, iv := range b.runs() { - n += a.bitmapCountRange(int32(iv.Start), int32(iv.Last)+1) + n += BitmapCountRange(a.bitmap(), int32(iv.Start), int32(iv.Last)+1) } return n } @@ -4370,6 +4368,7 @@ func unionArrayArray(a, b *Container) *Container { break } } + // note: len(output) CAN be > 4096 return NewContainerArray(output) } @@ -6867,8 +6866,12 @@ func ConvertRunToBitmap(c *Container) { func Optimize(c *Container) { c.optimize() } -func Union(a, b *Container) *Container { - return union(a, b) +func Union(a, b *Container) (c *Container) { + c = union(a, b) + // c can be have arrays that are too big, and need + // to be optimized into raw bitmaps. + c.optimize() + return c } func Difference(a, b *Container) *Container { diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index a3ae31527..2041af9ff 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -131,7 +131,7 @@ func TestContainerRunAdd2(t *testing.T) { func TestRunCountRange(t *testing.T) { c := NewContainerRun(nil) - cnt := c.runCountRange(2, 9) + cnt := RunCountRange(c.runs(), 2, 9) if cnt != 0 { t.Fatalf("should get 0 from empty container, but got: %v", cnt) } @@ -139,7 +139,7 @@ func TestRunCountRange(t *testing.T) { c.add(6) c.add(7) - cnt = c.runCountRange(2, 9) + cnt = RunCountRange(c.runs(), 2, 9) if cnt != 3 { t.Fatalf("should get 3 from interval within range, but got: %v", cnt) } @@ -149,52 +149,52 @@ func TestRunCountRange(t *testing.T) { c.add(10) c.add(11) - cnt = c.runCountRange(4, 8) + cnt = RunCountRange(c.runs(), 4, 8) if cnt != 3 { t.Fatalf("should get 3 from range overlaps front of interval, but got: %v", cnt) } - cnt = c.runCountRange(5, 8) + cnt = RunCountRange(c.runs(), 5, 8) if cnt != 3 { t.Fatalf("should get 3 from range within interval, but got: %v", cnt) } - cnt = c.runCountRange(6, 8) + cnt = RunCountRange(c.runs(), 6, 8) if cnt != 2 { t.Fatalf("should get 2 from range within interval, but got: %v", cnt) } - cnt = c.runCountRange(3, 9) + cnt = RunCountRange(c.runs(), 3, 9) if cnt != 4 { t.Fatalf("should get 4 from range overlaps front of interval, but got: %v", cnt) } - cnt = c.runCountRange(9, 14) + cnt = RunCountRange(c.runs(), 9, 14) if cnt != 3 { t.Fatalf("should get 3 from range overlaps back of interval, but got: %v", cnt) } - cnt = c.runCountRange(8, 10) + cnt = RunCountRange(c.runs(), 8, 10) if cnt != 2 { t.Fatalf("should get 2 from range within interval, but got: %v", cnt) } - cnt = c.runCountRange(8, 11) + cnt = RunCountRange(c.runs(), 8, 11) if cnt != 3 { t.Fatalf("should get 3 from range within interval, but got: %v", cnt) } - cnt = c.runCountRange(8, 12) + cnt = RunCountRange(c.runs(), 8, 12) if cnt != 4 { t.Fatalf("should get 4 from range overlaps back of interval, but got: %v", cnt) } - cnt = c.runCountRange(5, 12) + cnt = RunCountRange(c.runs(), 5, 12) if cnt != 7 { t.Fatalf("should get 7 from interval within range, but got: %v", cnt) } - cnt = c.runCountRange(5, 11) + cnt = RunCountRange(c.runs(), 5, 11) if cnt != 6 { t.Fatalf("should get 6 from interval equal to range, but got: %v", cnt) } @@ -203,7 +203,7 @@ func TestRunCountRange(t *testing.T) { c.add(19) c.add(18) - cnt = c.runCountRange(1, 22) + cnt = RunCountRange(c.runs(), 1, 22) if cnt != 10 { t.Fatalf("should get 10 from multiple ranges in interval, but got: %v", cnt) } @@ -211,7 +211,7 @@ func TestRunCountRange(t *testing.T) { c.add(13) c.add(14) - cnt = c.runCountRange(6, 18) + cnt = RunCountRange(c.runs(), 6, 18) if cnt != 9 { t.Fatalf("should get 9 from multiple ranges overlapping both sides, but got: %v", cnt) } @@ -263,7 +263,7 @@ func TestBitmapCountRange(t *testing.T) { for i, test := range tests { c.setBitmap(test.bitmap[:]) - if ret := c.bitmapCountRange(test.start, test.end); ret != test.exp { + if ret := BitmapCountRange(c.bitmap(), test.start, test.end); ret != test.exp { t.Fatalf("test #%v count of %v from %v to %v should be %v but got %v", i, test.bitmap, test.start, test.end, test.exp, ret) } } @@ -4404,3 +4404,40 @@ func TestUnionRunRunInPlaceBitwiseCompare(t *testing.T) { } } } + +func TestCloneRoaringIterator(t *testing.T) { + + ca := NewContainerArray([]uint16{1, 10, 100, 1000}) + ba := NewFileBitmap() + ba.Containers.Put(0, ca) + ba.Containers.Put(10, ca) + ba.Containers.Put(101, ca) + ba.Containers.Put(10001, ca) + var buf bytes.Buffer + _, err := ba.WriteTo(&buf) + if err != nil { + t.Fatalf("error writing: %v", err) + } + + itr, err := NewRoaringIterator(buf.Bytes()) + if err != nil { + t.Fatalf("error NewRoaringIterator(buf.Bytes()): %v", err) + } + + itr2 := itr.Clone() + + var keys []uint64 + for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { + keys = append(keys, itrKey) + _ = synthC + } + + var keys2 []uint64 + for itrKey, synthC := itr2.NextContainer(); synthC != nil; itrKey, synthC = itr2.NextContainer() { + keys2 = append(keys2, itrKey) + _ = synthC + } + if !reflect.DeepEqual(keys, keys2) { + t.Fatalf("keys != keys2. keys='%#v'; keys2='%#v'", keys, keys2) + } +} diff --git a/row.go b/row.go index 30c11d2fe..ac0ef56e3 100644 --- a/row.go +++ b/row.go @@ -18,7 +18,6 @@ import ( "encoding/json" "sort" - "github.com/molecula/ext" pb "github.com/pilosa/pilosa/v2/proto" "github.com/pilosa/pilosa/v2/roaring" "github.com/pkg/errors" @@ -326,65 +325,6 @@ func (r *Row) Union(others ...*Row) *Row { return &Row{segments: output} } -// GenericBinaryOp returns the output of a generic op on r and other. -func (r *Row) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *Row, args map[string]interface{}) *Row { - var segments []rowSegment - itr := newMergeSegmentIterator(r.segments, other.segments) - for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() { - if s1 == nil { - segments = append(segments, *s0) - continue - } else if s0 == nil { - segments = append(segments, *s1) - continue - } - segments = append(segments, *s0.GenericBinaryOp(op, s1, args)) - } - - return &Row{segments: segments} -} - -// GenericNaryOp returns the output of an nary op on r and others. -func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args map[string]interface{}) *Row { - segments := make([][]rowSegment, 0, len(others)+1) - if len(r.segments) > 0 { - segments = append(segments, r.segments) - } - nextSegs := make([][]rowSegment, 0, len(others)+1) - toProcess := make([]*rowSegment, 0, len(others)+1) - var output []rowSegment - for _, other := range others { - if len(other.segments) > 0 { - segments = append(segments, other.segments) - } - } - for len(segments) > 0 { - shard := segments[0][0].shard - for _, segs := range segments { - if segs[0].shard < shard { - shard = segs[0].shard - } - } - nextSegs = nextSegs[:0] - toProcess := toProcess[:0] - for _, segs := range segments { - if segs[0].shard == shard { - toProcess = append(toProcess, &segs[0]) - segs = segs[1:] - } - if len(segs) > 0 { - nextSegs = append(nextSegs, segs) - } - } - // at this point, "toProcess" is a list of all the segments - // sharing the lowest ID, and nextSegs is a list of all the others. - // Swap the segment lists (so we don't have to reallocate it) - segments, nextSegs = nextSegs, segments - output = append(output, *toProcess[0].GenericNaryOp(op, toProcess[1:], args)) - } - return &Row{segments: output} -} - // Difference returns the diff of r and other. func (r *Row) Difference(others ...*Row) *Row { var output []rowSegment @@ -408,17 +348,6 @@ func (r *Row) Difference(others ...*Row) *Row { return &Row{segments: output} } -// GenericUnaryOp returns the results of a generic op on r. -func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row { - work := r - var segments []rowSegment - for _, segment := range work.segments { - opped := segment.GenericUnaryOp(op, args) - segments = append(segments, *opped) - } - return &Row{segments: segments} -} - // Shift returns the bitwise shift of r by n bits. // Currently only positive shift values are supported. // @@ -523,15 +452,6 @@ func (r *Row) Count() uint64 { return n } -// GenericCount applies an op to lots of things. -func (r *Row) GenericCount(op ext.BitmapOpUnaryCount, args map[string]interface{}) uint64 { - var n int64 - for i := range r.segments { - n += op([]ext.Bitmap{WrapBitmap(r.segments[i].data)}, args) - } - return uint64(n) -} - // MarshalJSON returns a JSON-encoded byte slice of r. func (r *Row) MarshalJSON() ([]byte, error) { var o struct { @@ -644,33 +564,6 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment { } } -// GenericOp performs a generic op on s and other -func (s *rowSegment) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *rowSegment, args map[string]interface{}) *rowSegment { - data := op([]ext.Bitmap{WrapBitmap(s.data), WrapBitmap(other.data)}, args) - - return &rowSegment{ - data: UnwrapBitmap(data), - shard: s.shard, - n: data.Count(), - } -} - -// GenericOp performs a generic op on s and others -func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSegment, args map[string]interface{}) *rowSegment { - bitmaps := make([]ext.Bitmap, len(others)+1) - bitmaps[0] = WrapBitmap(s.data) - for i, seg := range others { - bitmaps[i+1] = WrapBitmap(seg.data) - } - data := op(bitmaps, args) - - return &rowSegment{ - data: UnwrapBitmap(data), - shard: s.shard, - n: data.Count(), - } -} - // Difference returns the diff of s and other. func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment { datas := make([]*roaring.Bitmap, len(others)) @@ -713,17 +606,6 @@ func (s *rowSegment) Shift() (*rowSegment, error) { }, nil } -// GenericUnaryOp returns s subject to op. -func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment { - data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args)) - - return &rowSegment{ - data: data, - shard: s.shard, - n: data.Count(), - } -} - // SetBit sets the i-th column of the row. func (s *rowSegment) SetBit(i uint64) (changed bool) { s.ensureWritable() diff --git a/server.go b/server.go index 5274c865a..2b501df6b 100644 --- a/server.go +++ b/server.go @@ -27,13 +27,11 @@ import ( "sync" "time" - "github.com/molecula/ext" uuid "github.com/satori/go.uuid" // extensions pulls in some extensions depending on build tags _ "github.com/pilosa/pilosa/v2/extensions" "github.com/pilosa/pilosa/v2/logger" - "github.com/pilosa/pilosa/v2/pql" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/stats" "github.com/pkg/errors" @@ -63,7 +61,6 @@ type Server struct { // nolint: maligned hosts []string clusterDisabled bool serializer Serializer - extensions []*ext.ExtensionInfo // External systemInfo SystemInfo @@ -430,30 +427,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.cluster.confirmDownRetries = s.confirmDownRetries s.cluster.confirmDownSleep = s.confirmDownSleep s.holder.broadcaster = s - err = s.loadAllExtensions() - if err != nil { - s.logger.Printf("not all plugins loaded successfully") - } - if len(s.extensions) > 0 { - s.logger.Printf("loaded extensions:") - for _, ext := range s.extensions { - if ext == nil { - s.logger.Printf(" inexplicably, a nil extension?!?") - continue - } - s.logger.Printf(" %s %s: %s", ext.Name, ext.Version, ext.Description) - if ext.License != "" { - s.logger.Printf(" License: %s", ext.License) - } - if len(ext.BitmapOps) > 0 { - opList := make([]string, len(ext.BitmapOps)) - for i := range ext.BitmapOps { - opList[i] = ext.BitmapOps[i].Name - } - s.logger.Printf(" Ops: %s", strings.Join(opList, ", ")) - } - } - } err = s.cluster.setup() if err != nil { @@ -467,56 +440,6 @@ func (s *Server) InternalClient() InternalClient { return s.defaultClient } -// loadNewExtensions loads extensions that have been -// registered since the last call to loadNewExtensions. -func (s *Server) loadNewExtensions() error { //nolint:unused - return s.loadExtensions(ext.NewExtensions()) -} - -// loadAllExtensions loads all extensions. -func (s *Server) loadAllExtensions() error { - return s.loadExtensions(ext.AllExtensions()) -} - -func (s *Server) loadExtensions(exts []*ext.ExtensionInfo) error { - var lastError error - for _, extension := range exts { - if err := s.loadExtension(extension); err != nil { - lastError = err - } - } - return lastError -} - -func (s *Server) loadExtension(extInfo *ext.ExtensionInfo) error { - if extInfo.ExtensionAPI != "v0" { - return fmt.Errorf("%s: unsupported extension API %s", extInfo.Name, extInfo.ExtensionAPI) - } - s.extensions = append(s.extensions, extInfo) - bitmapOps := extInfo.BitmapOps - bmOps, countOps, fieldOps, unknownOps := 0, 0, 0, 0 - for i := range bitmapOps { - typ := bitmapOps[i].Func.BitmapOpType() - switch { - case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount: - countOps++ - case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap: - bmOps++ - case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap: - fieldOps++ - default: - unknownOps++ - } - } - err := s.executor.registerOps(bitmapOps) - if err != nil { - s.logger.Printf("warning: extension registration failed: %v", err) - } else { - pql.RegisterPluginFuncs(bitmapOps) - } - return nil -} - // UpAndDown brings the server up minimally and shuts it down // again; basically, it exists for testing holder open and close. func (s *Server) UpAndDown() error { @@ -545,8 +468,12 @@ func (s *Server) UpAndDown() error { func (s *Server) Open() error { s.logger.Printf("open server") - // Start background monitoring. - s.snapshotQueue = newSnapshotQueue(10, 2, s.logger) + if s.holder.NeedsSnapshot() { + // Start background monitoring. + s.snapshotQueue = newSnapshotQueue(10, 2, s.logger) + } else { + s.snapshotQueue = defaultSnapshotQueue //TODO (twg) rethink this + } // Log startup err := s.holder.logStartup() @@ -711,6 +638,7 @@ func (s *Server) monitorAntiEntropy() { // the cluster sets its state to resizing and *then* sends to // abortAntiEntropyCh before starting to resize } + // Sync holders. s.logger.Printf("holder sync beginning") s.cluster.muAntiEntropy.Lock() diff --git a/server/cluster_test.go b/server/cluster_test.go index d788de803..744b63dcb 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -143,9 +143,9 @@ func TestClusterResize_AddNode(t *testing.T) { clus := test.MustRunCluster(t, 2) defer clus.Close() - if !checkClusterState(clus[0], pilosa.ClusterStateNormal, 1000) { + if !test.CheckClusterState(clus[0], pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State()) - } else if !checkClusterState(clus[1], pilosa.ClusterStateNormal, 1000) { + } else if !test.CheckClusterState(clus[1], pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", clus[1].API.State()) } }) @@ -176,9 +176,9 @@ func TestClusterResize_AddNode(t *testing.T) { } defer m1.Close() - if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) - } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + } else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } }) @@ -224,9 +224,9 @@ func TestClusterResize_AddNode(t *testing.T) { } defer m1.Close() - if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) - } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + } else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } @@ -273,9 +273,9 @@ func TestClusterResize_AddNode(t *testing.T) { } defer m1.Close() - if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) - } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + } else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } @@ -326,9 +326,9 @@ func TestClusterResize_AddNode(t *testing.T) { } defer m1.Close() - if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) - } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + } else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } @@ -373,9 +373,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { } defer m1.Close() - if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) - } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + } else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } @@ -431,9 +431,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { }() defer m1.Close() - if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) - } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + } else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } @@ -489,9 +489,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { } defer m1.Close() - if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) - } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + } else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } @@ -545,9 +545,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { } defer m1.Close() - if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) - } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + } else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) } m0.QueryExpect(t, "i", "", `Row(f=1)`, exp) @@ -598,11 +598,11 @@ func TestCluster_GossipMembership(t *testing.T) { t.Fatal(err) } - if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) - } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + } else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) - } else if !checkClusterState(m2, pilosa.ClusterStateNormal, 1000) { + } else if !test.CheckClusterState(m2, pilosa.ClusterStateNormal, 1000) { t.Fatalf("unexpected node2 cluster state: %s", m2.API.State()) } @@ -725,15 +725,3 @@ func TestClusterMutualTLS(t *testing.T) { t.Fatal(err) } } - -// checkClusterState polls a given cluster for its state until it -// receives a matching state. It polls up to n times before returning. -func checkClusterState(m *test.Command, state string, n int) bool { - for i := 0; i < n; i++ { - if m.API.State() == state { - return true - } - time.Sleep(10 * time.Millisecond) - } - return false -} diff --git a/test/cluster.go b/test/cluster.go index babb3890f..4fbc2050e 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -193,6 +193,18 @@ func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Clu return c } +// CheckClusterState polls a given cluster for its state until it +// receives a matching state. It polls up to n times before returning. +func CheckClusterState(m *Command, state string, n int) bool { + for i := 0; i < n; i++ { + if m.API.State() == state { + return true + } + time.Sleep(10 * time.Millisecond) + } + return false +} + // newCluster creates a new cluster func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluster, error) { if size == 0 { diff --git a/test/index.go b/test/index.go index b90702ae2..e51850d87 100644 --- a/test/index.go +++ b/test/index.go @@ -32,7 +32,9 @@ func newIndex() *Index { if err != nil { panic(err) } - index, err := pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i") + h := pilosa.NewHolder(pilosa.DefaultPartitionN) + h.Path = path + index, err := h.CreateIndex("i", pilosa.IndexOptions{}) if err != nil { panic(err) } @@ -42,7 +44,7 @@ func newIndex() *Index { // MustOpenIndex returns a new, opened index at a temporary path. Panic on error. func MustOpenIndex() *Index { index := newIndex() - if err := index.Open(); err != nil { + if err := index.Open(false); err != nil { panic(err) } return index @@ -62,12 +64,14 @@ func (i *Index) Reopen() error { } path, name := i.Path(), i.Name() - i.Index, err = pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, name) + h := pilosa.NewHolder(pilosa.DefaultPartitionN) + h.Path = h.HolderPathFromIndexPath(path, name) + i.Index, err = h.CreateIndex(name, pilosa.IndexOptions{}) if err != nil { return err } - if err := i.Open(); err != nil { + if err := i.Open(false); err != nil { return err } return nil diff --git a/translator_test.go b/translator_test.go index 72a4e8ddd..0acce9b66 100644 --- a/translator_test.go +++ b/translator_test.go @@ -296,6 +296,82 @@ func TestTranslation_Reset(t *testing.T) { }) } +// Test index key translation replication under node failure. +func TestTranslation_Replication(t *testing.T) { + t.Run("Replication", func(t *testing.T) { + c := test.MustRunCluster(t, 3, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerIsCoordinator(true), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(2), + )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerIsCoordinator(false), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(2), + )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerIsCoordinator(false), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(2), + )}, + ) + + node0 := c[0] + node1 := c[1] + + ctx := context.Background() + idx := "i" + field := "f" + + // Create an index with keys. + if _, err := node0.API.CreateIndex(ctx, idx, + pilosa.IndexOptions{ + Keys: true, + }); err != nil { + t.Fatal(err) + } + + if _, err := node0.API.CreateField(ctx, idx, field); err != nil { + t.Fatal(err) + } + + // Write data on first node. + // these keys are a minimal example to reproduce the problem for the case of a 3-node cluster with replication factor 2 + if _, err := node0.Queryf(t, idx, "", ` + Set("x1", f=1) + Set("x2", f=1) + `); err != nil { + t.Fatal(err) + } + + exp := `{"results":[{"attrs":{},"columns":[],"keys":["x1","x2"]}]}` + + if !test.CheckClusterState(node0, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node0 cluster state: %s", node0.API.State()) + } else if !test.CheckClusterState(node1, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node1 cluster state: %s", node1.API.State()) + } + + // Verify the data exists + node0.QueryExpect(t, idx, "", `Row(f=1)`, exp) + + // Kill one node. + if err := node1.Command.Close(); err != nil { + t.Fatal(err) + } + + // Verify the data exists with one node down + node0.QueryExpect(t, idx, "", `Row(f=1)`, exp) + }) +} + // Test key translation with multiple nodes. func TestTranslation_Coordinator(t *testing.T) { // Ensure that field key translations requests sent to diff --git a/tx.go b/tx.go index 9fcf6fd99..cc78b501f 100644 --- a/tx.go +++ b/tx.go @@ -15,9 +15,16 @@ package pilosa import ( + "bytes" "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" "sync" + "github.com/pilosa/pilosa/v2/rbf" "github.com/pilosa/pilosa/v2/roaring" "github.com/pkg/errors" ) @@ -50,6 +57,10 @@ const writable = true // that have not been committed. type Tx interface { + // Type returns "roaring", "rbf", "badger", "badger_roaring", or one of the other + // blue-green Tx types at the top of txfactory.go + Type() string + // Rollback must be called the end of read-only transactions. Either // Rollback or Commit must be called at the end of writable transactions. // It is safe to call Rollback multiple times, but it must be @@ -97,6 +108,11 @@ type Tx interface { // Calling Next() on the returned roaring.ContainerIterator gives // you a roaring.Container that is either run, array, or raw bitmap. // Return value 'found' is true when the ckey container was present. + // ckey of 0 gives all containers (in the fragment). + // + // ContainerIterator must not have side-effects. blueGreenTx will + // call it at the very beginning of commit to verify db contents. + // ContainerIterator(index, field, view string, shard uint64, ckey uint64) (citer roaring.ContainerIterator, found bool, err error) // RoaringBitmap retreives the roaring.Bitmap for the entire shard. @@ -162,10 +178,57 @@ type Tx interface { OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) // ImportRoaringBits does efficient bulk import using rit, a roaring.RoaringIterator. + // // See the roaring package for details of the RoaringIterator. + // // If clear is true, the bits from rit are cleared, otherwise they are set in the // specifed fragment. - ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) + // + // The data argument can be nil, its ignored for RBF/BadgerTx. It is supplied to + // RoaringTx.ImportRoaringBits() in fragment.go fragment.fillFragmentFromArchive() + // to do the traditional fragment.readStorageFromArchive() which + // does some in memory field/view/fragment metadata updates. + // It makes blueGreenTx testing viable too. + // + // ImportRoaringBits return values changed and rowSet may be inaccurate if + // the data []byte is supplied (the RoaringTx implementation neglects this for speed). + ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) + + RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) + + // SliceOfShards returns all of the shards for the specified index, field, view triple. + // Use within pilosa supposes a new read-only transaction was created just + // for the SliceOfShards() call. The legacy RoaringTx version is the only + // one that needs optionalViewPath; any other Tx implementation can ignore that. + SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) + + // Dump is for debugging, what does this Tx see as its database? + Dump() +} + +// TxStore has operations that will create and commit multiple +// Tx on a backing store. +type TxStore interface { + + // DeleteFragment deletes all the containers in a fragment. + // + // This is not in a Tx because it will often do too many deletes for a single + // transaction, and clients would be suprised to find their Tx had already + // been commited and they are getting an error on double-Commit. + // Instead each TxStore implementation creates and commits as many + // transactions as needed. + // + // Argument frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil. + // If not nil, it must be of type *fragment. If frag is supplied, then + // index must be equal to frag.index, field equal to frag.field, view equal + // to frag.view, and shard equal to frag.shard. + // + DeleteFragment(index, field, view string, shard uint64, frag interface{}) error + + DeleteField(index, field string) error + + // Close shuts down the database. + Close() error } // RawRoaringData used by ImportRoaringBits. @@ -207,10 +270,39 @@ func NewMultiTxWithIndex(writable bool, index *Index) *MultiTx { var _ Tx = (*MultiTx)(nil) +func (mtx *MultiTx) Type() string { + return RoaringTxn +} + +// debugging, what does this Tx see as its database? +func (mtx *MultiTx) Dump() { + mtx.mu.Lock() + defer mtx.mu.Unlock() + if len(mtx.txs) == 0 { + return + } + for _, tx := range mtx.txs { + tx.Dump() + return + } +} + +func (mtx *MultiTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { + tx, err := mtx.txNoShard(index) + panicOn(err) + return tx.SliceOfShards(index, field, view, optionalViewPath) +} + func (mtx *MultiTx) UseRowCache() bool { return true } +func (mtx *MultiTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { + tx, err := mtx.tx(index, shard) + panicOn(err) + return tx.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) +} + func (mtx *MultiTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { tx, err := mtx.tx(index, shard) panicOn(err) @@ -226,8 +318,10 @@ func (mtx *MultiTx) Pointer() string { return fmt.Sprintf("%p", mtx) } -func (tx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { - panic("not done") +func (mtx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { + tx, err := mtx.tx(index, shard) + panicOn(err) + return tx.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) } func (mtx *MultiTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { @@ -412,6 +506,20 @@ func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) { return tx, nil } +// version of the above for SliceOfShards(), where we don't have a shard. +func (mtx *MultiTx) txNoShard(index string) (_ Tx, err error) { + mtx.mu.Lock() + defer mtx.mu.Unlock() + + // Lookup transaction from cache. + for _, tx := range mtx.txs { + if tx.(*RoaringTx).Index.name == index { + return tx, nil + } + } + panic(fmt.Sprintf("no prior RoaringTx available, looking up index='%v'", index)) +} + type multiTxKey struct { index string shard uint64 @@ -426,10 +534,51 @@ type RoaringTx struct { fragment *fragment } +func (tx *RoaringTx) Type() string { + return RoaringTxn +} + +func (tx *RoaringTx) Dump() { + fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys()) +} + func (tx *RoaringTx) UseRowCache() bool { return true } +func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { + + // SliceOfShards is based on view.openFragments() + + file, err := os.Open(filepath.Join(optionalViewPath, "fragments")) + if os.IsNotExist(err) { + return + } else if err != nil { + return nil, errors.Wrap(err, "opening fragments directory") + } + defer file.Close() + + fis, err := file.Readdir(0) + if err != nil { + return nil, errors.Wrap(err, "reading fragments directory") + } + + for _, fi := range fis { + if fi.IsDir() { + continue + } + // Parse filename into integer. + shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) + if err != nil { + //AlwaysPrintf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name()) + //v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name()) + continue + } + sliceOfShards = append(sliceOfShards, shard) + } + return +} + func (tx *RoaringTx) Pointer() string { return fmt.Sprintf("%p", tx) } @@ -442,13 +591,24 @@ func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roa return b.Iterator() } -func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { - b, err := tx.bitmap(index, field, view, shard) - panicOn(err) +// ImportRoaringBits return values changed and rowSet will be inaccurate if +// the data []byte is supplied. This mimics the traditional roaring-per-file +// and should be faster. +func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { + f, err := tx.getFragment(index, field, view, shard) if err != nil { return 0, nil, err } - return b.ImportRoaringRawIterator(rit, clear, true, rowSize) + if len(data) > 0 { + // changed and rowSet are ignored anyway when len(data) > 0; + // when we are called from fragment.fillFragmentFromArchive() + // which is the only place the data []byte is supplied. + // blueGreenTx also turns off the checks in this case. + return 0, nil, f.readStorageFromArchive(bytes.NewBuffer(data)) + } + + changed, rowSet, err = f.storage.ImportRoaringRawIterator(rit, clear, true, rowSize) + return } func (tx *RoaringTx) Readonly() bool { @@ -625,8 +785,16 @@ func (tx *RoaringTx) OffsetRange(index, field, view string, shard uint64, offset // getFragment is used by IncrementOpN() and by bitmap() func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*fragment, error) { - // If a fragment is attached, always use it. + // If a fragment is attached, always use it. Since it was set at Tx creation, + // it is highly likely to be correct. if tx.fragment != nil { + // but still a basic sanity check. + if tx.fragment.index != index || + tx.fragment.field != field || + tx.fragment.view != view || + tx.fragment.shard != shard { + panic(fmt.Sprintf("different fragment cached vs requested. tx.fragment='%#v', index='%v', field='%v'; view='%v'; shard='%v'", tx.fragment, index, field, view, shard)) + } return tx.fragment, nil } @@ -659,8 +827,10 @@ func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*frag } frag := v.Fragment(shard) + if frag == nil { - panic(fmt.Sprintf("fragment not found: %q / %q / %d", field, view, shard)) + return nil, fmt.Errorf("fragment not found: %q / %q / %d", field, view, shard) + //panic(fmt.Sprintf("fragment not found: %q / %q / %d", field, view, shard)) } // Note: we cannot cache frag into tx.fragment. @@ -677,3 +847,212 @@ func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.B } return frag.storage, nil } + +type RoaringStore struct{} + +func NewRoaringStore() *RoaringStore { + return &RoaringStore{} +} + +func (db *RoaringStore) Close() error { + return nil +} + +func (db *RoaringStore) DeleteField(index, field, fieldPath string) error { + + // under blue-green badger_roaring, the directory will not be found, b/c badger will have + // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: + // "If the path does not exist, RemoveAll returns nil (no error)" + err := os.RemoveAll(fieldPath) + if err != nil { + return errors.Wrap(err, "removing directory") + } + return nil +} + +// frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil. +func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { + + fragment, ok := frag.(*fragment) + if !ok { + return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag) + } + + // Close data files before deletion. + if err := fragment.Close(); err != nil { + return errors.Wrap(err, "closing fragment") + } + + // Delete fragment file. + if err := os.Remove(fragment.path); err != nil { + return errors.Wrap(err, "deleting fragment file") + } + + // Delete fragment cache file. + if err := os.Remove(fragment.cachePath()); err != nil { + return errors.Wrap(err, fmt.Sprintf("no cache file to delete for shard %d", fragment.shard)) + } + return nil +} + +func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { + file, err := os.Open(fragmentPathForRoaring) // open the fragment file + if err != nil { + return nil, -1, err + } + fi, err := file.Stat() + if err != nil { + return nil, -1, errors.Wrap(err, "statting") + } + sz = fi.Size() + r = file + return +} + +type RBFTx struct { + index string + tx *rbf.Tx +} + +func (tx *RBFTx) Type() string { + return RBFTxn +} + +func (tx *RBFTx) Rollback() { + tx.tx.Rollback() +} + +func (tx *RBFTx) Commit() error { + return tx.tx.Commit() +} + +func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { + return tx.tx.RoaringBitmap(rbfName(field, view, shard)) +} + +func (tx *RBFTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) { + return tx.tx.Container(rbfName(field, view, shard), key) +} + +func (tx *RBFTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error { + return tx.tx.PutContainer(rbfName(field, view, shard), key, c) +} + +func (tx *RBFTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error { + return tx.tx.RemoveContainer(rbfName(field, view, shard), key) +} + +func (tx *RBFTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) { + return tx.tx.Add(rbfName(field, view, shard), a...) +} + +func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { + return tx.tx.Remove(rbfName(field, view, shard), a...) +} + +func (tx *RBFTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) { + return tx.tx.Contains(rbfName(field, view, shard), v) +} + +func (tx *RBFTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) { + return tx.tx.ContainerIterator(rbfName(field, view, shard), key) +} + +func (tx *RBFTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { + return tx.tx.ForEach(rbfName(field, view, shard), fn) +} + +func (tx *RBFTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { + return tx.tx.ForEachRange(rbfName(field, view, shard), start, end, fn) +} + +func (tx *RBFTx) Count(index, field, view string, shard uint64) (uint64, error) { + return tx.tx.Count(rbfName(field, view, shard)) +} + +func (tx *RBFTx) Max(index, field, view string, shard uint64) (uint64, error) { + return tx.tx.Max(rbfName(field, view, shard)) +} + +func (tx *RBFTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { + return tx.tx.Min(rbfName(field, view, shard)) +} + +func (tx *RBFTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { + return tx.tx.UnionInPlace(rbfName(field, view, shard), others...) +} + +func (tx *RBFTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) { + return tx.tx.CountRange(rbfName(field, view, shard), start, end) +} + +func (tx *RBFTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) { + return tx.tx.OffsetRange(rbfName(field, view, shard), offset, start, end) +} + +func (tx *RBFTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {} + +func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { + return tx.tx.ImportRoaringBits(rbfName(field, view, shard), rit, clear, log, rowSize, data) +} + +func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { + panic("TODO: Implement RBFTx.RoaringBitmapReader()") +} + +func (tx *RBFTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { + prefix := rbfFieldViewPrefix(field, view) + + names, err := tx.tx.BitmapNames() + if err != nil { + return nil, err + } + + // Iterate over shard names and collect shards from matching field/view prefix. + for _, name := range names { + if !strings.HasPrefix(name, prefix) { + continue + } + + s := strings.TrimPrefix(name, prefix) + shard, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return nil, errors.Wrap(err, "parse shard id from rbf key") + } + sliceOfShards = append(sliceOfShards, shard) + } + return sliceOfShards, nil +} + +func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { + b, err := tx.RoaringBitmap(index, field, view, shard) + panicOn(err) + return b.Iterator() +} + +func (tx *RBFTx) Pointer() string { + return fmt.Sprintf("%p", tx) +} + +func (tx *RBFTx) Dump() { + tx.tx.Dump(tx.index) +} + +// Readonly is true if the transaction is not read-and-write, but only doing reads. +func (tx *RBFTx) Readonly() bool { + return !tx.tx.Writable() +} + +func (tx *RBFTx) UseRowCache() bool { + return false +} + +// rbfName returns a NULL-separated key used for identifying bitmap maps in RBF. +func rbfName(field, view string, shard uint64) string { + return fmt.Sprintf("%s\x00%s\x00%d", field, view, shard) +} + +// rbfFieldViewPrefix returns a NULL-separated prefix for keys in RBF. +func rbfFieldViewPrefix(field, view string) string { + return fmt.Sprintf("%s\x00%s\x00", field, view) +} diff --git a/txfactory.go b/txfactory.go index f8af55c6c..de2e90c25 100644 --- a/txfactory.go +++ b/txfactory.go @@ -21,7 +21,9 @@ import ( "strconv" "strings" "syscall" + "text/tabwriter" + "github.com/pilosa/pilosa/v2/rbf" "github.com/pilosa/pilosa/v2/roaring" "github.com/pkg/errors" ) @@ -55,13 +57,15 @@ var sep = string(os.PathSeparator) type TxFactory struct { typeOfTx txtype - bw *BadgerDBWrapper + badgerDB *BadgerDBWrapper + + rbfDB *rbf.DB + + roaringDB *RoaringStore // could have more than one *Index, but for now keep it simple, // and allow blueGreenTx to report badger contents via idx idx *Index - - // TODO: put RBF database handle here. } // integer types for fast switch{} @@ -85,6 +89,32 @@ const ( blueGreenRBFBadger txtype = 9 ) +func (txf *TxFactory) NeedsSnapshot() bool { + switch txf.typeOfTx { + case noneTxn: + panic("noneTxn should not occur") + case roaringFragmentFilesTxn: + return true + case badgerTxn: + return false + case rbfTxn: + return false + case blueGreenBadgerRoaring: + return true + case blueGreenRoaringBadger: + return true + case blueGreenRBFRoaring: + return true + case blueGreenRoaringRBF: + return true + case blueGreenBadgerRBF: + return false + case blueGreenRBFBadger: + return false + } + panic(fmt.Sprintf("unknown typeOfTx '%v'", txf.typeOfTx)) +} + func MustTxsrcToTxtype(txsrc string) txtype { switch txsrc { case RoaringTxn: // "roaring" @@ -109,29 +139,55 @@ func MustTxsrcToTxtype(txsrc string) txtype { panic(fmt.Sprintf("unknown txsrc '%v'", txsrc)) } -func newTxFactory(txsrc string, path string) (f *TxFactory, err error) { +// always store files in a subdir of dir. If we are having one +// database or many can depend on name. +func NewTxFactory(txsrc string, dir, name string, openExisting bool) (f *TxFactory, err error) { + ty := MustTxsrcToTxtype(txsrc) if ty < 1 || ty > 9 { panic(fmt.Sprintf("invalid txtype '%v'", int(ty))) } - var bw *BadgerDBWrapper - if ty == badgerTxn || ty == 4 || ty == 5 || ty == 8 || ty == 9 { - bw, err = openBadgerDBWrapper(path) - // TODO(jea): figure out what the appropriate error path is here. - //fmt.Printf("warning: could not open badgerdb on path '%v': '%v'. For safety, we are opening a new '%v-fallback' instead\n", path, err, path+"-fallback") - if err != nil { - //bw, err = newBadgerDBWrapper(path + "-fallback") - bw, err = newBadgerDBWrapper(path) - } - panicOn(err) - - bw.doAllocZero = true + f = &TxFactory{ + typeOfTx: ty, + roaringDB: NewRoaringStore(), } - return &TxFactory{ - typeOfTx: ty, - bw: bw, - }, err + + switch ty { + case badgerTxn, blueGreenBadgerRoaring, blueGreenRoaringBadger, blueGreenBadgerRBF, blueGreenRBFBadger: + + // one, big, bad-ass badger for all data: the honeyBadger. + // + // Note that having a single Tx backing store for all indexes + // enables cross-index Tx, which are important and are tested for. + path := dir + sep + "honeyBadger" + + if openExisting { + f.badgerDB, err = globalBadgerReg.openBadgerDBWrapper(path) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("cannot open badger db. path='%v'", path)) + } + } else { + f.badgerDB, err = globalBadgerReg.newBadgerDBWrapper(path) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("cannot create new badger db. path='%v'", path)) + } + } + // electric-fence like finding of access to mmapped data beyond + // transaction end time. + f.badgerDB.doAllocZero = true + } + + switch ty { + case rbfTxn, blueGreenRBFRoaring, blueGreenRoaringRBF, blueGreenBadgerRBF, blueGreenRBFBadger: + + f.rbfDB = rbf.NewDB(filepath.Join(dir, "db.rbf")) + if err := f.rbfDB.Open(); err != nil { + return nil, errors.Wrap(err, "cannot open rbf db") + } + } + + return f, err } // Txo holds the transaction options @@ -153,72 +209,151 @@ func (f *TxFactory) DeleteIndex(name string) error { // from holder.go:955, by default is already done there with os.RemoveAll() return nil case badgerTxn: - return f.bw.DeleteIndex(name) + return f.badgerDB.DeleteIndex(name) case rbfTxn: panic("todo rbfTxn DeleteIndex(name)") case blueGreenBadgerRoaring: - return f.bw.DeleteIndex(name) + return f.badgerDB.DeleteIndex(name) case blueGreenRoaringBadger: - return f.bw.DeleteIndex(name) + return f.badgerDB.DeleteIndex(name) } panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx)) } -func (f *TxFactory) Close() error { +func (f *TxFactory) DeleteFieldFromStore(index, field, fieldPath string) error { switch f.typeOfTx { case roaringFragmentFilesTxn: - return nil + return f.roaringDB.DeleteField(index, field, fieldPath) case badgerTxn: - // note cannot actually close Badger here. - // causes problems b/c tries holder.DeleteIndex tries to delete the index after db is closed. - //return f.bw.Close() - return nil + return f.badgerDB.DeleteField(index, field, fieldPath) case rbfTxn: - panic("todo rbfTxn Close()") + //return f.rbfDB.DeleteField(index, field, fieldPath) + return nil case blueGreenBadgerRoaring: - return nil + _ = f.badgerDB.DeleteField(index, field, fieldPath) + return f.roaringDB.DeleteField(index, field, fieldPath) case blueGreenRoaringBadger: - return nil + _ = f.roaringDB.DeleteField(index, field, fieldPath) + return f.badgerDB.DeleteField(index, field, fieldPath) } panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx)) } +func (f *TxFactory) DeleteFragmentFromStore(index, field, view string, shard uint64, frag *fragment) error { + switch f.typeOfTx { + case roaringFragmentFilesTxn: + return f.roaringDB.DeleteFragment(index, field, view, shard, frag) + case badgerTxn: + return f.badgerDB.DeleteFragment(index, field, view, shard, frag) + case rbfTxn: + tx, err := f.rbfDB.Begin(true) + if err != nil { + return err + } + defer tx.Rollback() + + if err := tx.DeleteBitmapsWithPrefix(rbfFieldViewPrefix(field, view)); err != nil { + return err + } + return tx.Commit() + case blueGreenBadgerRoaring: + _ = f.badgerDB.DeleteFragment(index, field, view, shard, frag) + return f.roaringDB.DeleteFragment(index, field, view, shard, frag) + case blueGreenRoaringBadger: + _ = f.roaringDB.DeleteFragment(index, field, view, shard, frag) + return f.badgerDB.DeleteFragment(index, field, view, shard, frag) + } + panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx)) + +} + func (f *TxFactory) CloseIndex(idx *Index) error { switch f.typeOfTx { case roaringFragmentFilesTxn: return nil case badgerTxn: + // note cannot actually close Badger here. + // causes problems b/c tries holder.DeleteIndex tries to delete the index after db is closed. + //return f.badgerDB.Close() return nil case rbfTxn: - panic("todo rbfTxn CloseIndex()") - + return f.rbfDB.Close() case blueGreenBadgerRoaring: return nil case blueGreenRoaringBadger: return nil + + case blueGreenRBFRoaring: + _ = f.rbfDB.Close() + return nil + case blueGreenRoaringRBF: + return f.rbfDB.Close() + case blueGreenBadgerRBF: + return f.rbfDB.Close() + case blueGreenRBFBadger: + _ = f.rbfDB.Close() + return nil + } panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx)) } func (f *TxFactory) NewTx(o Txo) Tx { + indexName := "" + if o.Index != nil { + indexName = o.Index.name + } switch f.typeOfTx { case roaringFragmentFilesTxn: return &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment} case badgerTxn: - btx := f.bw.NewBadgerTx(o.Write) + btx := f.badgerDB.NewBadgerTx(o.Write, indexName) return btx case rbfTxn: - panic("todo rbfTxn creation") - + tx, err := f.rbfDB.Begin(o.Write) + if err != nil { + panic(err) // TODO: Add error return on NewTx() + } + return &RBFTx{tx: tx, index: indexName} case blueGreenBadgerRoaring: - btx := f.bw.NewBadgerTx(o.Write) + btx := f.badgerDB.NewBadgerTx(o.Write, indexName) rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment} return newBlueGreenTx(btx, rtx, f.idx) case blueGreenRoaringBadger: - btx := f.bw.NewBadgerTx(o.Write) + btx := f.badgerDB.NewBadgerTx(o.Write, indexName) rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment} return newBlueGreenTx(rtx, btx, f.idx) + + case blueGreenBadgerRBF: + btx := f.badgerDB.NewBadgerTx(o.Write, indexName) + rbftx, err := f.rbfDB.Begin(o.Write) + if err != nil { + panic(errors.Wrap(err, "rbfDB.Begin transaction errored")) + } + return newBlueGreenTx(btx, &RBFTx{tx: rbftx, index: indexName}, f.idx) + case blueGreenRBFBadger: + btx := f.badgerDB.NewBadgerTx(o.Write, indexName) + rbftx, err := f.rbfDB.Begin(o.Write) + if err != nil { + panic(errors.Wrap(err, "rbfDB.Begin transaction errored")) + } + return newBlueGreenTx(&RBFTx{tx: rbftx, index: indexName}, btx, f.idx) + + case blueGreenRBFRoaring: + rbftx, err := f.rbfDB.Begin(o.Write) + if err != nil { + panic(errors.Wrap(err, "rbfDB.Begin transaction errored")) + } + rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment} + return newBlueGreenTx(&RBFTx{tx: rbftx, index: indexName}, rtx, f.idx) + case blueGreenRoaringRBF: + rbftx, err := f.rbfDB.Begin(o.Write) + if err != nil { + panic(errors.Wrap(err, "rbfDB.Begin transaction errored")) + } + rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment} + return newBlueGreenTx(rtx, &RBFTx{tx: rbftx, index: indexName}, f.idx) } panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx)) } @@ -255,14 +390,13 @@ func (ty txtype) String() string { // Hence to view uncommited keys, you must provide in optionalUseThisTx the // Tx in which they have been added. func (idx *Index) StringifiedBadgerKeys(optionalUseThisTx Tx) string { - return idx.Txf.bw.StringifiedBadgerKeys(optionalUseThisTx) + return idx.Txf.badgerDB.StringifiedBadgerKeys(optionalUseThisTx) } // fragmentSpecFromRoaringPath takes a path releative to the // index directory, not including the name of the index itself. // The path should not start with the path separator sep ('/' or '\\') rune. func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64, err error) { - if len(path) == 0 { err = fmt.Errorf("fragmentSpecFromRoaringPath error: path '%v' too short", path) return @@ -291,29 +425,38 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64, } func (idx *Index) StringifiedRoaringKeys() (r string) { - paths, err := listFilesUnderDir(idx.path, false, "", true) panicOn(err) index := idx.name r = "allkeys:[\n" + n := 0 for _, relpath := range paths { field, view, shard, err := fragmentSpecFromRoaringPath(relpath) if err != nil { continue // ignore .meta paths } abspath := idx.path + sep + relpath - s, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard) + const showOps = false + s, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard, showOps) panicOn(err) //r += fmt.Sprintf("path:'%v' fragment contains:\n") + s + if s == "" { + s = "" + } r += s + n++ } + if n == 0 { + return "" // new convention that empty database => empty string returned. + } + // note that we can have a bitmap present, but it can be empty r += "]\n all-in-blake3:" + blake3sum16([]byte(r)) + "\n" return "roaring-" + r } -func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64) (r string, err error) { +func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64, showOps bool) (r string, err error) { var info roaring.BitmapInfo _ = info @@ -352,6 +495,21 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard return } + //cmd.DisplayInfo(info) + // inlined + if showOps { + pC := pointerContext{ + from: info.From, + to: info.To, + } + if info.ContainerCount > 0 { + printContainers(info, pC) + } + if info.Ops > 0 { + printOps(info) + } + } + citer, found := rbm.Containers.Iterator(0) _ = found // probably gonna use just the Ops log instead, so don't panic if !found. @@ -432,28 +590,6 @@ func fileSize(name string) (int64, error) { var _ = fileSize // happy linter -// Dump prints to stdout the contents of the roaring Containers -// stored in idx. Its format may vary depending of the type of -// idx.Txf transaction factory that is in use. -// Mostly for debugging. -func (idx *Index) Dump(label string) { - ty := idx.Txf.TxType() - fileline := FileLine(2) - switch ty { - case badgerTxn: - fmt.Printf("%v Index.Dump('%v') for index '%v':\n%v\n", fileline, label, idx.name, idx.StringifiedBadgerKeys(nil)) - return - case blueGreenRoaringBadger, blueGreenBadgerRoaring: - fmt.Printf("%v Index.Dump('%v') for index '%v', RoaringTx:\n%v\n", fileline, label, idx.name, idx.StringifiedRoaringKeys()) - fmt.Printf("%v Index.Dump('%v') for index '%v', BadgerTx :\n%v\n", fileline, label, idx.name, idx.StringifiedBadgerKeys(nil)) - return - case roaringFragmentFilesTxn: - fmt.Printf("%v Index.Dump('%v') for index '%v', BadgerTx :\n%v\n", fileline, label, idx.name, idx.StringifiedRoaringKeys()) - return - } - panic(fmt.Errorf("%v Index.Dump('%v') for index '%v': no implementation for txtype '%v'\n", fileline, label, idx.name, ty)) -} - func containerToBytes(ct *roaring.Container) []byte { ty := roaring.ContainerType(ct) switch ty { @@ -468,3 +604,109 @@ func containerToBytes(ct *roaring.Container) []byte { } panic(fmt.Sprintf("unknown container type '%v'", int(ty))) } + +type pointerContext struct { + from, to uintptr +} + +func printOps(info roaring.BitmapInfo) { + fmt.Fprintln(os.Stdout, " Ops:") + tw := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0) + fmt.Fprintf(tw, " \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE") + printed := 0 + for _, op := range info.OpDetails { + fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size) + printed++ + } + tw.Flush() +} + +func (p *pointerContext) pretty(c roaring.ContainerInfo) string { + var pointer string + if c.Mapped { + if c.Pointer >= p.from && c.Pointer < p.to { + pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from) + } else { + pointer = fmt.Sprintf("!0x%x!", c.Pointer) + } + } else { + pointer = fmt.Sprintf("0x%x", c.Pointer) + } + return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer) +} + +// stolen from ctl/inspect.go +func printContainers(info roaring.BitmapInfo, pC pointerContext) { + fmt.Fprintln(os.Stdout, " Containers:") + tw := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0) + fmt.Fprintf(tw, " \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n") + fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS") + c1s := info.Containers + c2s := info.OpContainers + l1 := len(c1s) + l2 := len(c2s) + i1 := 0 + i2 := 0 + var c1, c2 roaring.ContainerInfo + c1.Key = ^uint64(0) + c2.Key = ^uint64(0) + c1e := false + c2e := false + if i1 < l1 { + c1 = c1s[i1] + i1++ + c1e = true + } + if i2 < l2 { + c2 = c2s[i2] + i2++ + c2e = true + } + printed := 0 + for c1e || c2e { + c1used := false + c2used := false + var key uint64 + c1fmt := "-\t\t\t" + c2fmt := "-\t\t\t" + // If c2 exists, we'll always prefer its flags, + // if it doesn't, this gets overwritten. + flags := c2.Flags + if !c2e || (c1e && c1.Key < c2.Key) { + c1fmt = pC.pretty(c1) + key = c1.Key + c1used = true + flags = c1.Flags + } else if !c1e || (c2e && c2.Key < c1.Key) { + c2fmt = pC.pretty(c2) + key = c2.Key + c2used = true + } else { + // c1e and c2e both set, and neither key is < the other. + c1fmt = pC.pretty(c1) + c2fmt = pC.pretty(c2) + key = c1.Key + c1used = true + c2used = true + } + if c1used { + if i1 < l1 { + c1 = c1s[i1] + i1++ + } else { + c1e = false + } + } + if c2used { + if i2 < l2 { + c2 = c2s[i2] + i2++ + } else { + c2e = false + } + } + fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags) + printed++ + } + tw.Flush() +} diff --git a/utils_internal_test.go b/utils_internal_test.go index e9603b222..f24fded67 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -15,8 +15,6 @@ package pilosa import ( - "bufio" - "bytes" "fmt" "io/ioutil" "path/filepath" @@ -396,9 +394,8 @@ func (b bcast) SendTo(to *Node, m Message) error { return nil } -// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing. +// FollowResizeInstruction is a version of cluster.followResizeInstruction used for testing. func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error { - // Prepare the return message. complete := &ResizeInstructionComplete{ JobID: instr.JobID, @@ -420,7 +417,8 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error } // Sync available shards. - for _, is := range instr.NodeStatus.Indexes { + for k, is := range instr.NodeStatus.Indexes { + _ = k for _, fs := range is.Fields { f := destCluster.holder.Field(is.Name, fs.Name) @@ -451,23 +449,29 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error } } - buf := bytes.NewBuffer(nil) + // this is the *test* version of a network call, transferring fragments between + // nodes in a cluster. So it is allowed to be kind of a hack. - bw := bufio.NewWriter(buf) - br := bufio.NewReader(buf) + // there will be two -badgerdb directories/databases, we need to copy + // from src to dest the fragment. This simulates sending the fragment over the network. + srcIdx := srcCluster.holder.Index(src.Index) + srctx := srcIdx.Txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment}) - // Get the fragment from source. - if _, err := srcFragment.WriteTo(bw); err != nil { - return err - } - - // Flush the bufio.buf to the io.Writer (buf). - bw.Flush() - - // Write data to destination. - if _, err := destFragment.ReadFrom(br); err != nil { - return err + destIdx := destCluster.holder.Index(src.Index) + + desttx := destIdx.Txf.NewTx(Txo{Write: writable, Index: destIdx, Fragment: destFragment}) + + citer, _, err := srctx.ContainerIterator(src.Index, src.Field, src.View, src.Shard, 0) + panicOn(err) + d := destFragment + for citer.Next() { + ckey, c := citer.Value() + err := desttx.PutContainer(d.index, d.field, d.view, d.shard, ckey, c) + panicOn(err) } + citer.Close() + panicOn(desttx.Commit()) + srctx.Rollback() } return nil diff --git a/view.go b/view.go index 12210a619..430845fbb 100644 --- a/view.go +++ b/view.go @@ -50,6 +50,7 @@ type view struct { qualifiedName string holder *Holder + idx *Index fieldType string cacheType string @@ -136,14 +137,18 @@ func (v *view) open() error { if err := func() error { // Ensure the view's path exists. v.holder.Logger.Debugf("ensure view path exists: %s", v.path) - if err := os.MkdirAll(v.path, 0777); err != nil { + err := os.MkdirAll(v.path, 0777) + if err != nil { return errors.Wrap(err, "creating view directory") - } else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil { + } + err = os.MkdirAll(filepath.Join(v.path, "fragments"), 0777) + if err != nil { return errors.Wrap(err, "creating fragments directory") } v.holder.Logger.Debugf("open fragments for index/field/view: %s/%s/%s", v.index, v.field, v.name) - if err := v.openFragments(); err != nil { + + if err := v.openFragmentsInTx(); err != nil { return errors.Wrap(err, "opening fragments") } @@ -159,49 +164,46 @@ func (v *view) open() error { var workQueue = make(chan struct{}, runtime.NumCPU()*2) -// openFragments opens and initializes the fragments inside the view. -func (v *view) openFragments() error { - file, err := os.Open(filepath.Join(v.path, "fragments")) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return errors.Wrap(err, "opening fragments directory") - } - defer file.Close() +// replaces v.openFragments() with Tx generic code. +func (v *view) openFragmentsInTx() error { - fis, err := file.Readdir(0) + tx := v.idx.Txf.NewTx(Txo{Write: !writable, Index: v.idx}) + defer tx.Rollback() + + shards, err := tx.SliceOfShards(v.index, v.field, v.name, v.path) if err != nil { - return errors.Wrap(err, "reading fragments directory") + return errors.Wrap(err, "SliceOfShards") } eg, ctx := errgroup.WithContext(context.Background()) var mu sync.Mutex -fileLoop: - for _, loopFi := range fis { + shardCh := make(chan uint64, len(shards)) + for i := range shards { + shardCh <- shards[i] + } + +shardLoop: + for range shards { select { case <-ctx.Done(): - break fileLoop + break shardLoop default: - fi := loopFi - - if fi.IsDir() { - continue - } - - // Parse filename into integer. - shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64) - if err != nil { - v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name()) - continue - } workQueue <- struct{}{} - v.holder.Logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard) eg.Go(func() error { defer func() { <-workQueue }() + + var shard uint64 + select { + case shard = <-shardCh: + default: + return nil // no more work + } + v.holder.Logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard) + frag := v.newFragment(v.fragmentPath(shard), shard) if err := frag.Open(); err != nil { return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err) @@ -359,6 +361,16 @@ func (v *view) notifyIfNewShard(shard uint64) { } func (v *view) newFragment(path string, shard uint64) *fragment { + + if v.holder != nil && v.idx != nil { + // A view must have its v.idx *Index registered with its holder. + // Otherwise TestField_AvailableShards crashes, as one example. + hIdx := v.holder.Index(v.idx.name) + if hIdx == nil && v.idx != nil { + v.holder.addIndexFromField(v.idx) + } + } + frag := newFragment(v.holder, path, v.index, v.field, v.name, shard, v.flags()) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize @@ -375,28 +387,17 @@ func (v *view) newFragment(path string, shard uint64) *fragment { func (v *view) deleteFragment(shard uint64) error { v.mu.Lock() defer v.mu.Unlock() - fragment := v.fragments[shard] - if fragment == nil { + f := v.fragments[shard] + if f == nil { return ErrFragmentNotFound } v.holder.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard) - // Close data files before deletion. - if err := fragment.Close(); err != nil { - return errors.Wrap(err, "closing fragment") + idx := f.holder.Index(v.index) + if err := idx.Txf.DeleteFragmentFromStore(f.index, f.field, f.view, f.shard, f); err != nil { + return errors.Wrap(err, "DeleteFragment") } - - // Delete fragment file. - if err := os.Remove(fragment.path); err != nil { - return errors.Wrap(err, "deleting fragment file") - } - - // Delete fragment cache file. - if err := os.Remove(fragment.cachePath()); err != nil { - v.holder.Logger.Printf("no cache file to delete for shard %d", shard) - } - delete(v.fragments, shard) v.removeKnownShard(shard) diff --git a/view_internal_test.go b/view_internal_test.go index bbb1a20f4..bcde5f871 100644 --- a/view_internal_test.go +++ b/view_internal_test.go @@ -34,7 +34,15 @@ func mustOpenView(index, field, name string) *view { CacheSize: DefaultCacheSize, } - v := newView(NewHolder(DefaultPartitionN), path, index, field, name, fo) + h := NewHolder(DefaultPartitionN) + h.Path = path + // h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment + idx, err := h.createIndex(index, IndexOptions{}) + _ = idx + panicOn(err) + + v := newView(h, path, index, field, name, fo) + v.idx = idx if err := v.open(); err != nil { panic(err) } diff --git a/vprint.go b/vprint.go index bc40dfe45..05159615b 100644 --- a/vprint.go +++ b/vprint.go @@ -111,3 +111,57 @@ func FileLine(depth int) string { func stack() string { return string(debug.Stack()) } + +func FileExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return false + } + return true +} + +func DirExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return true + } + return false +} + +func FileSize(name string) (int64, error) { + fi, err := os.Stat(name) + if err != nil { + return -1, err + } + return fi.Size(), nil +} + +// Caller returns the name of the calling function. +func Caller(upStack int) string { + // elide ourself and runtime.Callers + target := upStack + 2 + + pc := make([]uintptr, target+2) + n := runtime.Callers(0, pc) + + f := runtime.Frame{Function: "unknown"} + if n > 0 { + frames := runtime.CallersFrames(pc[:n]) + for i := 0; i <= target; i++ { + contender, more := frames.Next() + if i == target { + f = contender + } + if !more { + break + } + } + } + return f.Function +} diff --git a/xrbrsupport.go b/xrbrsupport.go index ebfd9db27..70720582a 100644 --- a/xrbrsupport.go +++ b/xrbrsupport.go @@ -63,7 +63,7 @@ func (rbc *RBFConverter) Convert(index, field, view string, shard uint64, rb *ro if err != nil { return err } - defer func() { _ = tx.Rollback() }() + defer tx.Rollback() name := fmt.Sprintf("%s/%s", field, view) err = tx.CreateBitmap(name)