diff --git a/.circleci/config.yml b/.circleci/config.yml index 16180ae16..3a54b16b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -201,6 +201,7 @@ workflows: - test: name: test-shardwidth-22 shard_width: "22" + resource_class: large requires: - setup - cluster-tests: diff --git a/Makefile b/Makefile index b082fceeb..65840bf4d 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ RELEASE_ENABLED = $(subst 0,,$(RELEASE)) NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" ) BUILD_TAGS += $(if $(RELEASE_ENABLED),release) BUILD_TAGS += shardwidth$(SHARD_WIDTH) +TEST_TAGS = roaringparanoia define LICENSE_HASH_CODE head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " " endef @@ -38,11 +39,11 @@ vendor: go.mod # Run test suite test: - go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) + go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v # Run test suite with race flag test-race: - go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 60m -v + go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 60m -v testv: topt testvsub @@ -57,7 +58,7 @@ testvsub: set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i"; \ cd $$i; pwd; \ - go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -timeout 60m || break; \ + go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -timeout 60m || break; \ echo; echo "999 done testing subpkg $$i"; \ cd ..; \ done @@ -66,7 +67,7 @@ testvsub-race: set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i -race"; \ cd $$i; pwd; \ - go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race -timeout 60m || break; \ + go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race -timeout 60m || break; \ echo; echo "999 done testing subpkg $$i -race"; \ cd ..; \ done @@ -194,55 +195,55 @@ pilosa-chk: # Run Pilosa tests inside Docker container docker-test: - docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./... + docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) ./... # run top tests, not subdirs. print summary red/green after. # The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt. topt: mv log.topt.roar log.topt.roar.prev || true - go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.roar + go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.roar @echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l @echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l topt-badger: mv log.topt.badger log.topt.badger.prev || true - PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger + PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger @echo " log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l @echo " log.topt.badger red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l topt-badger-race: mv log.topt.badger-race log.topt.badger-race.prev || true - PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger-race + PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger-race @echo " log.topt.badger-race green: \c"; cat log.topt.badger-race | grep PASS |wc -l @echo " log.topt.badger-race red: \c"; cat log.topt.badger-race | grep '\-\-\- FAIL' |wc -l topt-rbf: mv log.topt.rbf log.topt.rbf.prev || true - PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.rbf + PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.rbf @echo " log.topt.rbf green: \c"; cat log.topt.rbf | grep PASS |wc -l @echo " log.topt.rbf red: \c"; cat log.topt.rbf | grep '\-\-\- FAIL' |wc -l topt-rbf-race: mv log.topt.rbf-race log.topt.rbf-race.prev || true - PILOSA_TXSRC=rbf go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -timeout 120m 2>&1 | tee log.topt.rbf-race + PILOSA_TXSRC=rbf go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -timeout 120m 2>&1 | tee log.topt.rbf-race @echo " log.topt.rbf-race green: \c"; cat log.topt.rbf-race | grep PASS |wc -l @echo " log.topt.rbf-race red: \c"; cat log.topt.rbf-race | grep '\-\-\- FAIL' |wc -l topt-lmdb: mv log.topt.lmdb log.topt.lmdb.prev || true - PILOSA_TXSRC=lmdb go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lmdb + PILOSA_TXSRC=lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lmdb @echo " log.topt.lmdb green: \c"; cat log.topt.lmdb | grep PASS |wc -l @echo " log.topt.lmdb red: \c"; cat log.topt.lmdb | grep '\-\-\- FAIL' |wc -l topt-lmdb-race: mv log.topt.lmdb log.topt.lmdb.prev || true - PILOSA_TXSRC=lmdb go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lmdb-race + PILOSA_TXSRC=lmdb go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lmdb-race @echo " log.topt.lmdb-race green: \c"; cat log.topt.lmdb-race | grep PASS |wc -l @echo " log.topt.lmdb-race red: \c"; cat log.topt.lmdb-race | grep '\-\-\- FAIL' |wc -l topt-race: mv log.topt.race log.topt.race.prev || true - go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.race + go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.race @echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l @echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l @@ -251,73 +252,73 @@ topt-race: bg-rr: # shorthand for bluegreen test with A:badger; B:roaring mv log.bg-rr log.bg-rr.prev || true - PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rr + PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rr @echo " log.bg-rr green: \c"; cat log.bg-rr | grep PASS |wc -l @echo " log.bg-rr red: \c"; cat log.bg-rr | grep '\-\-\- FAIL' |wc -l rr-bg: # bluegreen with A:roaring; B:badger (B's values are returned). mv log.bg.roar_bg log.bg.roar_bg.prev || true - PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-bg + PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-bg @echo " log.rr-bg green: \c"; cat log.rr-bg | grep PASS |wc -l @echo " log.rr-bg red: \c"; cat log.rr-bg | grep '\-\-\- FAIL' |wc -l rbf-rr: mv log.rbf-rr log.rbf-rr.prev || true - PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-rr + PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-rr @echo " log.rbf-rr green: \c"; cat log.rbf-rr | grep PASS |wc -l @echo " log.rbf-rr red: \c"; cat log.rbf-rr | grep '\-\-\- FAIL' |wc -l rr-rbf: mv log.rr-rbf log.rr-rbf.prev || true - PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-rbf + PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-rbf @echo " log.rr-rbf green: \c"; cat log.rr-rbf | grep PASS |wc -l @echo " log.rr-rbf red: \c"; cat log.rr-rbf | grep '\-\-\- FAIL' |wc -l rbf-bg: mv log.rbf-bg log.rbf-bg.prev || true - PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-bg + PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-bg @echo " log.rbf-bg green: \c"; cat log.rbf-bg | grep PASS |wc -l @echo " log.rbf-bg red: \c"; cat log.rbf-bg | grep '\-\-\- FAIL' |wc -l bg-rbf: mv log.bg-rbf log.bg-rbf.prev || true - PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rbf + PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rbf @echo " log.bg-rbf green: \c"; cat log.bg-rbf | grep PASS |wc -l @echo " log.bg-rbf red: \c"; cat log.bg-rbf | grep '\-\-\- FAIL' |wc -l rbf-lm: mv log.rbf-lm log.rbf-lm.prev || true - PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-lm + PILOSA_TXSRC=rbf_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-lm @echo " log.rbf-lm green: \c"; cat log.rbf-lm | grep PASS |wc -l @echo " log.rbf-lm red: \c"; cat log.rbf-lm | grep '\-\-\- FAIL' |wc -l lm-rbf: mv log.lm-rbf log.lm-rbf.prev || true - PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.lm-rbf + PILOSA_TXSRC=lmdb_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.lm-rbf @echo " log.lm-rbf green: \c"; cat log.lm-rbf | grep PASS |wc -l @echo " log.lm-rbf red: \c"; cat log.lm-rbf | grep '\-\-\- FAIL' |wc -l lm-rr: mv log.lm-rr log.lm-rr.prev || true - PILOSA_TXSRC=lmdb_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.lm-rr + PILOSA_TXSRC=lmdb_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.lm-rr @echo " log.lm-rr green: \c"; cat log.lm-rr | grep PASS |wc -l @echo " log.lm-rr red: \c"; cat log.lm-rr | grep '\-\-\- FAIL' |wc -l rr-lm: mv log.rr-lm log.rr-lm.prev || true - PILOSA_TXSRC=roaring_lmdb go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-lm + PILOSA_TXSRC=roaring_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-lm @echo " log.rr-lm green: \c"; cat log.rr-lm | grep PASS |wc -l @echo " log.rr-lm red: \c"; cat log.rr-lm | grep '\-\-\- FAIL' |wc -l bg-lm: mv log.topt.bg-lm log.topt.bg-lm.prev || true - PILOSA_TXSRC=badger_lmdb go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.bg-lm + PILOSA_TXSRC=badger_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.bg-lm @echo " log.topt.bg-lm green: \c"; cat log.topt.bg-lm | grep PASS |wc -l @echo " log.topt.bg-lm red: \c"; cat log.topt.bg-lm | grep '\-\-\- FAIL' |wc -l lm-bg: mv log.topt.lm-bg log.topt.lm-bg.prev || true - PILOSA_TXSRC=lmdb_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lm-bg + PILOSA_TXSRC=lmdb_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lm-bg @echo " log.topt.lm-bg green: \c"; cat log.topt.lm-bg | grep PASS |wc -l @echo " log.topt.lm-bg red: \c"; cat log.topt.lm-bg | grep '\-\-\- FAIL' |wc -l diff --git a/api.go b/api.go index 260c40c6f..f462830ff 100644 --- a/api.go +++ b/api.go @@ -54,6 +54,10 @@ type API struct { Serializer Serializer } +func (api *API) Holder() *Holder { + return api.holder +} + // apiOption is a functional option type for pilosa.API type apiOption func(*API) error diff --git a/badger.go b/badger.go new file mode 100644 index 000000000..66941bbb3 --- /dev/null +++ b/badger.go @@ -0,0 +1,1919 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "os" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + badger "github.com/dgraph-io/badger/v2" + badgeroptions "github.com/dgraph-io/badger/v2/options" + "github.com/pilosa/pilosa/v2/hash" + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/testhook" + "github.com/pilosa/pilosa/v2/txkey" + "github.com/pkg/errors" +) + +// TODO: is there a more optimal time to do badger garbage collection? +// As in: do we need to be more aggressive about cleaning in +// proportion to write activity? Space monitoring available with the +// badger.DB.Size() (lsm, vlog int64) call. +// +// See: https://godoc.org/github.com/dgraph-io/badger#DB.RunValueLogGC +// and: https://github.com/dgraph-io/badger#garbage-collection +// +// For now we run GC periodically every 1 minute or as set by the +// BadgerDBWrapper.GcEveryDur duration. +// +// Background: (quoting from docs referenced above) +// +// "Badger relies on the client to perform garbage collection at a time of +// their choosing. It provides the following method, which can be invoked +// at an appropriate time: +// +// "DB.RunValueLogGC(): This method is designed to do garbage collection while +// Badger is online. Along with randomly picking a file, it uses statistics +// generated by the LSM-tree compactions to pick files that are likely to +// lead to maximum space reclamation. It is recommended to be called during +// periods of low activity in your system, or periodically. One call would +// only result in removal of at max one log file. As an optimization, you +// could also immediately re-run it whenever it returns nil error (indicating +// a successful value log GC), as shown below." +// +// ticker := time.NewTicker(5 * time.Minute) +// defer ticker.Stop() +// for range ticker.C { +// again: +// err := db.RunValueLogGC(0.5) +// if err == nil { +// goto again +// } +// } +// + +// +// ========================================================= +// A note on using a recent version of badgerdb: +// +// We require a v2 release of badger after 2020 May 13, when support for +// multiple read-write iterators within one transaction was added. +// Many executor_test.go tests do foreachRow() operations, +// which call BadgerTx.ContainerIterator(), which in turn creates +// a first read-write iterator, and then OffsetRange(), which needs a +// second iterator, while still in the same read-write transaction. +// +// The most recent v2 master was pulled in and added to go.mod +// by doing go get github.com/dgraph-io/badger/v2@master +// resulting in the go.mod line +// github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 +// as of this writing, 2020 July 09. This version contains the support +// for having multiple read-write iterators. +// +// Reference on github.com/dgraph-io/badger +// +// commit af22dfd8d51317d765f0c05dcdf1d15981cca4f3 +// Author: Elliot Courant +// Date: Wed May 13 01:07:33 2020 -0500 +// +// Support multiple iterators in read-write transactions. (#1286) +// +// This adds support for multiple iterators during a read-write transaction. The +// iterators created in a read-write transaction will only be able to see writes +// that were performed before the iterator was created. Any writes that occur +// after the iterator is created will be invisible to the iterator. +// +// Fixes https://github.com/dgraph-io/badger/issues/981 +// +// +// Otherwise we'll get these panics: +// 'Only one iterator can be active at one time, for a RW txn.' +// when trying to open a second iterator on the same write transaction. +// e.g. go test -v -run TestExecutor_TranslateRowsOnBool + +var badgerDefaultLogger *BadgerLog +var badgerTestLogger *BadgerLog + +const BadgerLogToStderr = false + +func init() { + // badger test output clutters up the screen, dump to /dev/null for now. + null, err := os.Open(os.DevNull) + panicOn(err) + var out io.Writer = null + if BadgerLogToStderr { + // view badger logs + out = os.Stderr + } + badgerTestLogger = &BadgerLog{Logger: log.New(out, "badger ", log.LstdFlags)} + badgerDefaultLogger = badgerTestLogger + + // BadgerDB recommends a minimum of 128 GOMAXPROCS to make use of the IOPs + // available on the SSD. So we set that here. Details: + // + // from https://github.com/dgraph-io/badger#are-there-any-go-specific-settings-that-i-should-use + // + // "We *highly* recommend setting a high number for GOMAXPROCS, + // which allows Go to observe the full IOPS throughput provided by + // modern SSDs. In Dgraph, we have set it to 128. For more details, + // see this thread [https://groups.google.com/forum/#!topic/golang-nuts/jPb_h3TvlKE/discussion]." + // + // From that thread on golang-nuts: + // + // "Manish Rai Jain + // 8/7/17 + // Hey folks, + // During Gophercon, I happened to meet Russ Cox and asked him the same question. + // If File::Read blocks goroutines, which then spawn new OS threads, in a long running job, + // there should be plenty of OS threads created already, so the random read throughput + // should increase over time and stabilize to the maximum possible value. But, that's + // not what I see in my benchmarks. + // + // And his explanation was that the GOMAXPROCS in a way acts like a multiplexer. + // From docs, "the GOMAXPROCS variable limits the number of operating system threads + // that can execute user-level Go code simultaneously." Which basically means, all + // reads must first be run only via GOMAXPROCS number of goroutines, before switching + // over to some OS thread (not really a switch, but conceptually speaking). This + // introduces a bottleneck for throughput. + // I re-ran my benchmarks with a much higher GOMAXPROCS and was able to then + // achieve the maximum throughput. The numbers are here: + // https://github.com/dgraph-io/badger-bench/blob/master/randread/maxprocs.txt + // To summarize these benchmarks, Linux fio achieves 118K IOPS, and with GOMAXPROCS=64/128, + // I'm able to achieve 105K IOPS, which is close enough. Win! + // + // Regarding the point about using io_submit etc., instead of goroutines; I managed to + // find a library which does that, but it performed worse than just using goroutines. + // https://github.com/traetox/goaio/issues/3 + // From what I gather (talking to Russ and Ian), whatever work is going on in user space, + // the same work has to happen in kernel space; so there's not much benefit here. + // + // Overall, with GOMAXPROCS set to a higher value (as I've done in Dgraph), one can get + // the advertised SSD throughput using goroutines." + // + // moved to holder init + //runtime.GOMAXPROCS(128) +} + +// BadgerLog exists because badger requires a particular logger interface, with a +// Debugf method that is not on standard library log.Logger +type BadgerLog struct { + *log.Logger +} + +// Errorf logs an error. +func (l *BadgerLog) Errorf(f string, v ...interface{}) { + l.Printf("ERROR: "+f, v...) +} + +// Warningf logs a warning. +func (l *BadgerLog) Warningf(f string, v ...interface{}) { + l.Printf("WARNING: "+f, v...) +} + +// Infof logs an informational statement. +func (l *BadgerLog) Infof(f string, v ...interface{}) { + l.Printf("INFO: "+f, v...) +} + +// Debugf logs a debug statement. +func (l *BadgerLog) Debugf(f string, v ...interface{}) { + l.Printf("DEBUG: "+f, v...) +} + +// badgerRegistrar facilitates shutdown +// of all the badger databases started under +// tests. Its needed because most tests don't cleanup +// the *Index(es) they create. But we still +// want to shutdown badgerDB goroutines +// after tests run. +// +// It also allows opening the same path twice to +// result in sharing the same open database handle, and +// thus the same transactional guarantees. +// +type badgerRegistrar struct { + mu sync.Mutex + mp map[*BadgerDBWrapper]bool + + path2db map[string]*BadgerDBWrapper +} + +func (r *badgerRegistrar) Size() int { + r.mu.Lock() + defer r.mu.Unlock() + nmp := len(r.mp) + npa := len(r.path2db) + if nmp != npa { + panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa)) + } + return nmp +} + +var globalBadgerReg *badgerRegistrar = newBadgerTestRegistrar() + +func newBadgerTestRegistrar() *badgerRegistrar { + return &badgerRegistrar{ + mp: make(map[*BadgerDBWrapper]bool), + path2db: make(map[string]*BadgerDBWrapper), + } +} + +// register each badger created under tests, so we +// can clean them up. This is called by openBadgerDBWrapper() while +// holding the r.mu.Lock, since it needs to atomically +// check the registry and make a new instance only +// if one does not exist for its path, and otherwise +// return the existing instance. +func (r *badgerRegistrar) unprotectedRegister(w *BadgerDBWrapper) { + r.mp[w] = true + r.path2db[w.path] = w +} + +// unregister removes w from r +func (r *badgerRegistrar) unregister(w *BadgerDBWrapper) { + r.mu.Lock() + delete(r.mp, w) + delete(r.path2db, w.path) + r.mu.Unlock() +} + +func DumpAllBadger() { + globalBadgerReg.mu.Lock() + defer globalBadgerReg.mu.Unlock() + for w := range globalBadgerReg.mp { + _ = w + AlwaysPrintf("this badger path='%v' has: \n%v\n", w.path, w.StringifiedBadgerKeys(nil, true)) + } +} + +// badgerPath is a helper for determining the full directory +// in which the badger database will be stored. +func badgerPath(path string) string { + if !strings.HasSuffix(path, "-badgerdb@") { + return path + "-badgerdb@" + } + return path +} + +// openBadgerDB opens the database in the bpath directoy +// without deleting any prior content. Any BadgerDB +// database directory will have the "-badgerdb" suffix. +// +// openBadgerDB will check the registry and make a new instance only +// if one does not exist for its bpath. Otherwise it returns +// the existing instance. This insures only one badgerDB +// per bpath in this pilosa node. +func (r *badgerRegistrar) OpenDBWrapper(bpath string, doAllocZero bool) (DBWrapper, error) { + // now that newTxFactory can call us directly, we might not + // have the -badgerdb suffix. + bpath = badgerPath(bpath) + + r.mu.Lock() + defer r.mu.Unlock() + w, ok := r.path2db[bpath] + if ok { + // creates the effect of having only one badger open per pilosa node. + return w, nil + } + // otherwise, make a new badger and store it in globalBadgerReg + + // regular: works on amd64, but 386 doesn't work. + opt := badger.DefaultOptions(bpath).WithLogger(badgerDefaultLogger) + + opt.Compression = badgeroptions.None // turn off compression. + opt.ZSTDCompressionLevel = 0 // really, just in case. + opt.SyncWrites = true // default is true, safe. + //opt.KeepL0InMemory = true // speedup? + + // MaxCacheSize docs: + // + // how much data cache should hold in memory. A small size of + // cache means lower memory consumption and lookups/iterations + // would take longer. It is recommended to use a cache if you're + // using compression or encryption. If compression and + // encryption both are disabled, adding a cache will lead to + // unnecessary overhead which will affect the read performance. + // Setting size to zero disables the cache altogether. + //opt.MaxCacheSize = 1 << 30 // slows down 135 sec vs 113 sec on our benchmark + //opt.MaxCacheSize = 0 + opt.LoadBloomsOnOpen = false // should speed up start-up time. + + //opt.KeepBlocksInCache = true // default false + //opt.KeepBlockIndicesInCache = true // default false + + opt.BlockSize = 8 * 1024 // default 4 * 1024 + + // to get memory only do: + //opt := badger.DefaultOptions("").WithLogger(badgerDefaultLogger).WithInMemory(true) + + //db, err := badger.Open(opt) + db, err := badger.OpenManaged(opt) + if err != nil { + return nil, err + } + halt := make(chan bool) + w = &BadgerDBWrapper{ + reg: r, + path: bpath, + db: db, + halt: halt, + hasher: hash.NewBlake3Hasher(), + doAllocZero: doAllocZero, + } + _ = testhook.Opened(NewAuditor(), w, nil) + r.unprotectedRegister(w) + + w.startBadgerGarbageCollectionBackgroundGoro() + return w, nil +} + +func (w *BadgerDBWrapper) DeleteDBPath(dbs *DBShard) error { + panic("TODO") +} + +// DeleteIndex deletes all the containers associated with +// the named index from the badger database. +func (w *BadgerDBWrapper) DeleteIndex(indexName string) error { + + // We use the apostrophie rune `'` to locate the end of the + // index name in the key prefix, so we cannot allow indexNames + // themselves to contain apostrophies. + if strings.Contains(indexName, "'") { + return fmt.Errorf("error: bad indexName `%v` in BadgerDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes.", indexName) + } + prefix := txkey.IndexOnlyPrefix(indexName) + return w.DeletePrefix(prefix) +} + +// startBadgerGarbageCollectionBackgroundGoro handles Badger DB +// garbage colection by regularly purging the value log from +// a background goroutine. w.GcEveryDur controls how often +// it runs. The default is after every 60 seconds. +func (w *BadgerDBWrapper) startBadgerGarbageCollectionBackgroundGoro() { + go func() { + dur := w.GcEveryDur + if dur == 0 { + dur = time.Minute + } + ticker := time.NewTicker(dur) + defer ticker.Stop() + for { + select { + case <-ticker.C: + w.muGC.Lock() + again: + err := w.db.RunValueLogGC(0.5) + if err == nil { + goto again + } + w.muGC.Unlock() + case <-w.halt: + return + } + } + }() +} + +// statically confirm that BadgerTx satisfies the Tx interface. +var _ Tx = (*BadgerTx)(nil) + +var globalNextTxSnBadger int64 + +// BadgerDBWrapper provides the NewTx() method. +// The methods on BadgerDBWrapper are thread-safe, and can be called +// from different goroutines/threads. +type BadgerDBWrapper struct { + // serialize operations on BadgerDBWrapper and thus on the .db too, + // when obtaining new txns on different goroutines. + muDb sync.Mutex + + path string + db *badger.DB + + // track our registrar for Close / goro leak reporting purposes. + reg *badgerRegistrar + + // openTx and openIt are BadgerDBWrapper scoped tables of all open + // transactions and iterators. These are primarily for debugging purposes. + // openTx and openIt should only be read/written after locking the muOpenTxIt mutex. + + // the bool value is the writable attribute of the key *BadgerTx + openTx map[*BadgerTx]bool + + // the bool value is whether the iterator is reversed + openIt map[*BadgerIterator]bool + + // protect openTx and openIt + muOpenTxIt sync.Mutex + + // close(halt) to shutdown the badger gc goroutine in Close() + halt chan bool + + // make BadgerDBWrapper.Close() idempotent, avoiding panic on double Close() + closed bool + + // GcEveryDur controls how often the background goroutine + // runs garbage collection on the on-disk values-log. + // It defaults to running a GC every 1 minute if left as 0. + GcEveryDur time.Duration + + // muGC ensures we only run one Garbage Collection at a time. + muGC sync.Mutex + + hasher *hash.Blake3Hasher + + // doAllocZero sets the corresponding flag on all new BadgerTx. + // When doAllocZero is true, we zero out any data from badger + // after transcation commit and rollback. This simulates + // what would happen if we were to use the mmap-ed data + // from badger directly. Currently we copy by default for + // safety because otherwise TestAPI_ImportColumnAttrs sees + // corrupted data. + doAllocZero bool + + // stack() from our creation point, to track tests + // that haven't closed us. + startStack string + + DeleteEmptyContainer bool + + writeBatch *badger.WriteBatch +} + +func (w *BadgerDBWrapper) Path() string { + return w.path +} + +func (w *BadgerDBWrapper) SetHolder(h *Holder) { + // don't need it at the moment + //w.h = h +} + +// EntryCount for Badger returns a count of either 0 or 1. A 1 +// means it +func (w *BadgerDBWrapper) HasData() (has bool, err error) { + tx, err := w.NewTx(!writable, "", Txo{Write: false}) + if err != nil { + return false, err + } + defer tx.Rollback() + bi := NewBadgerIterator(tx.(*BadgerTx), nil) + defer bi.Close() + bi.Seek(nil) + if bi.it.Valid() { + return true, nil + } + return false, nil +} + +func (w *BadgerDBWrapper) CleanupTx(tx Tx) { + r := tx.(*BadgerTx) + r.mu.Lock() + defer r.mu.Unlock() + w.unprotectedCleanupTx(r) +} + +func (w *BadgerDBWrapper) unprotectedCleanupTx(btx *BadgerTx) { + if btx.done { + return + } + btx.done = true + btx.o.dbs.Cleanup(btx) // release the read/write lock. +} + +// unprotectedListOpenTxAsString is a debugging helper. +// It is not thread safe, but is only used for debugging. Called internally while +// holding locks. +func (w *BadgerDBWrapper) unprotectedListOpenTxAsString() (r string) { + + r = "openTx list = [" + for txn, write := range w.openTx { + r += fmt.Sprintf("txn p=%p(write:%v), ", txn, write) + } + return r + "]" +} + +var _ = (*BadgerDBWrapper)(nil).unprotectedListOpenTxAsString // linter happy + +// UnprotectedListOpenItAsString is exported because it is +// used for debugging in some of the pilosa_test tests. +// It is not thread safe, but only used for debugging. Called internally +// while holding locks and externally while not. +func (w *BadgerDBWrapper) UnprotectedListOpenItAsString() (r string) { + r = "openIt list = [" + for it, reverse := range w.openIt { + r += fmt.Sprintf("it p=%p(reverse:%v), ", it, reverse) + } + return r + "]" +} + +// NewTx produces BadgerDB based ACID transactions. If +// the transaction will modify data, then the write flag must be true. +// Read-only queries should set write to false, to allow more concurrency. +// Methods on a BadgerTx are thread-safe, and can be called from +// different goroutines. +// +// initialIndexName is optional. It is set by the TxFactory from the Txo +// options provided at the Tx creation point. It allows us to recognize +// and isolate cross-index queries more quickly. It can always be empty "" +// but when set is highly useful for debugging. It has no impact +// on transaction behavior. +// +func (w *BadgerDBWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { + + sn := atomic.AddInt64(&globalNextTxSnBadger, 1) + + tx = &BadgerTx{ + frag: o.Fragment, + write: write, + tx: w.db.NewTransactionAt(1, write), + Db: w, + doAllocZero: w.doAllocZero, + initialIndexName: initialIndexName, + DeleteEmptyContainer: w.DeleteEmptyContainer, + o: o, + sn: sn, + } + return +} + +// Close shuts down the Badger database. +func (w *BadgerDBWrapper) Close() (err error) { + + w.muDb.Lock() + defer w.muDb.Unlock() + if !w.closed { + w.reg.unregister(w) + close(w.halt) + w.closed = true + } + _ = testhook.Closed(NewAuditor(), w, nil) + return w.db.Close() +} + +// BadgerTx wraps a badger.Txn and provides the Tx interface +// method implementations. +// The methods on BadgerTx are thread-safe, and can be called +// from different goroutines. +type BadgerTx struct { + + // mu serializes badger operations on this single txn instance. + // + // reference: https://godoc.org/github.com/dgraph-io/badger + // "Running [two separate -jea] transactions concurrently is OK. However, a + // transaction itself isn't thread safe, and should only + // be run serially. It doesn't matter if a transaction is + // created by one goroutine and passed down to other, as + // long as the Txn APIs are called serially." + mu sync.Mutex + + write bool + Db *BadgerDBWrapper + tx *badger.Txn + + frag *fragment + opcount int + + doAllocZero bool + + // for tracking txn boundary issues, track all the memory + // that we deploy for roaring containers, and zero it on + // transaction commit/rollback. + acMu sync.Mutex // protect ourAllocs and ourContainers + ourAllocs [][]byte + ourContainers []*roaring.Container + + initialIndexName string + + DeleteEmptyContainer bool + + // We must avoid writing more than 10MB to badger in + // one transaction. If we go over, then + // we'll get a ErrTxnTooBig error. At that point + // we can't commit more, because the transaction + // will "conflict". So we must monitor + // totals written and auto-commit before going + // over the limits to avoid wedging into an + // unrecoverable state. + // + // We now use the approach described in https://github.com/dgraph-io/badger/issues/539 + // wherein "I fixed it by using a ManagedDB and using a commtTs of 1 always." + // But see also + // https://github.com/dgraph-io/badger/issues/409 + // wrt to "Write skew anomaly with concurrent transactions". + // which means we probably still only want one writeTx at a time + // even for badger. + writeCount int + writeByteCount int + + o Txo + sn int64 // serial number + + done bool +} + +func (tx *BadgerTx) IsDone() (done bool) { + tx.mu.Lock() + done = tx.done + tx.mu.Unlock() + return +} + +func (tx *BadgerTx) Group() *TxGroup { + return tx.o.Group +} + +func (tx *BadgerTx) Options() Txo { + return tx.o +} + +func (tx *BadgerTx) Type() string { + return BadgerTxn +} + +func (tx *BadgerTx) UseRowCache() bool { + //the row cache speeds up queries. + return false +} + +// overWriteOurAllocs provides detection of memory +// access outside the transactional context, similar to the +// old school electric fence techniques but without setting +// memory mappings to read-only... instead we just zero +// out the memory allocated to roaring containers by a +// transaction after the commit or rollback. This, +// hopefully, will cause some downstream confusion and +// test failures, which we can use to locate who has been +// holding on to memory they should have copied prior +// to transaction commit. +func (tx *BadgerTx) overWriteOurAllocs() { + + tx.acMu.Lock() + defer tx.acMu.Unlock() + for _, s := range tx.ourAllocs { + + // The Go compiler recognizes the following pattern and inserts + // an efficient memclr instruction. + // See https://github.com/golang/go/issues/5373 + // and https://codereview.appspot.com/137880043 + for i := range s { + s[i] = 0 + // or + // Seebs suggested we might see even more crashes :) + // but since it will be slow (no memclr), we'll leave the default 0 for now. + //s[i] = -2 + } + } + // keep this around if we need to activate out-of-mmap memory access again. + //for _, v := range tx.ourContainers { + //v.Invalid = true + //v.Tx = tx + //} +} + +// Pointer gives us a memory address for the underlying transaction for debugging. +// It is public because we use it in roaring to report invalid container memory access +// outside of a transaction. +func (tx *BadgerTx) Pointer() string { + return fmt.Sprintf("%p", tx) +} + +// Rollback rolls back the transaction. +func (tx *BadgerTx) Rollback() { + tx.mu.Lock() + defer tx.mu.Unlock() + + //pp("BadgerTx.Rollback p=%p, its: '%v' initloc: '%v',\n rollbackloc:'%v'", tx, tx.Db.UnprotectedListOpenItAsString(), tx.initloc, stack()) + tx.tx.Discard() // must hold tx.mu mutex lock + + tx.Db.muOpenTxIt.Lock() + delete(tx.Db.openTx, tx) + tx.Db.muOpenTxIt.Unlock() + + if tx.doAllocZero { + // and clear our allocs, to find code using them outside of a txn. + tx.overWriteOurAllocs() + } + + tx.Db.unprotectedCleanupTx(tx) +} + +// Commit commits the transaction to permanent storage. +// Commits can handle up to 100k updates to fragments +// at once, but not more. This is a BadgerDB imposed limit. +func (tx *BadgerTx) Commit() error { + tx.mu.Lock() + defer tx.mu.Unlock() + + tx.Db.muOpenTxIt.Lock() + delete(tx.Db.openTx, tx) + tx.Db.muOpenTxIt.Unlock() + + //pp("BadgerTx.Commit (write:%v) p=%p, stackID=%x openit: '%v' initloc: '%v', commitloc:\n%v", tx.write, tx, stackID, tx.Db.UnprotectedListOpenItAsString(), tx.initloc, stack()) + + err := tx.tx.CommitAt(1, nil) // must hold tx.mu mutex lock + + if tx.doAllocZero { + tx.overWriteOurAllocs() + } + + tx.Db.unprotectedCleanupTx(tx) + return err +} + +// Readonly returns true iff the BadgerTx is read-only. +func (tx *BadgerTx) Readonly() bool { + return !tx.write +} + +// RoaringBitmap returns the roaring.Bitmap for all bits in the fragment. +func (tx *BadgerTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { + + return tx.OffsetRange(index, field, view, shard, 0, 0, LeftShifted16MaxContainerKey) +} + +// Container returns the requested roaring.Container, selected by fragment and ckey +func (tx *BadgerTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) { + + // values returned from Get() are only valid while the transaction + // is open. If you need to use a value outside of the transaction then + // you must use copy() to copy it to another byte slice. + // BUT here we are already inside the Txn. + + bkey := txkey.Key(index, field, view, shard, ckey) + tx.mu.Lock() + var item *badger.Item + item, err = tx.tx.Get(bkey) + tx.mu.Unlock() + if err == badger.ErrKeyNotFound { + // Seems crazy, but we, for now at least, + // match what RoaringTx does by returning nil, nil. + return nil, nil + } else { + panicOn(err) + } + + err = item.Value(func(v []byte) error { + // This func with val would only be called if item.Value encounters no error + c = tx.toContainer(item.UserMeta(), v) + return nil + }) + panicOn(err) + return +} + +func (w *BadgerDBWrapper) NewWriteBatch() { + w.muDb.Lock() + defer w.muDb.Unlock() + if w.writeBatch != nil { + panic("must FlushWriteBatch() before calling NewWriteBatch()") + } + w.writeBatch = w.db.NewWriteBatch() +} + +// Flush any remaining un-committed writes in progress. +func (w *BadgerDBWrapper) FlushWriteBatch() (err error) { + w.muDb.Lock() + defer w.muDb.Unlock() + if w.writeBatch == nil { + panic("CommitWriteBatch error: no batch in progress") + } + err = w.writeBatch.Flush() + w.writeBatch = nil + return +} + +// Cancel any remaining un-committed writes in progress. +func (w *BadgerDBWrapper) CancelWriteBatch() { + w.muDb.Lock() + defer w.muDb.Unlock() + if w.writeBatch == nil { + panic("CancelWriteBatch error: no batch in progress") + } + w.writeBatch.Cancel() +} + +// PutContainer stores rc under the specified fragment and container ckey. +func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey uint64, rc *roaring.Container) error { + + bkey := txkey.Key(index, field, view, shard, ckey) + var by []byte + + ct := roaring.ContainerType(rc) + + switch ct { + case roaring.ContainerArray: + by = fromArray16(roaring.AsArray(rc)) + case roaring.ContainerBitmap: + by = fromArray64(roaring.AsBitmap(rc)) + case roaring.ContainerRun: + by = fromInterval16(roaring.AsRuns(rc)) + case roaring.ContainerNil: + panic("wat? nil roaring.Container is unexpected, no?!?") + default: + panic(fmt.Sprintf("unknown roaring.Container type: %v", ct)) + } + + entry := badger.NewEntry(bkey, by).WithMeta(ct) + + tx.Db.muDb.Lock() + if tx.Db.writeBatch != nil { + err := tx.Db.writeBatch.SetEntry(entry) // Will create txns as needed. + tx.Db.muDb.Unlock() + return err + } + tx.Db.muDb.Unlock() + + tx.mu.Lock() + defer tx.mu.Unlock() + + tx.writeCount++ + sz := len(by) + len(bkey) + 2 + tx.writeByteCount += sz + + // The integration tests do large bit level loads that exceed 10MB. + // So we autocommit and start a new Txn if we are about to + // write too much into one Txn. + // + // The badger defaults limits are currently: + // maxBatchCount:104857, maxBatchSize:10066329 + // + // However, emprirically we still get ErrTnTooBig when + // tx.writeByteCount=5884222; or when tx.writeCount=16197. + // So duck under both those thresholds by some margin. + if tx.writeCount > 100 || tx.writeByteCount > 2000000 { + // avoid ErrTxnTooBig by commiting before going over the limits, + // because then we get a error: "Transaction Conflict. Please retry." + err := tx.tx.CommitAt(1, nil) + panicOn(err) + // badger docs: + // `ErrConflict is returned when a transaction conflicts with another transaction. This can + // happen if the read rows had been updated concurrently by another transaction. + // ErrConflict = errors.New("Transaction Conflict. Please retry")` + //if err == badger.ErrConflict { + // problem is, we don't have the previous entry handy now. + //} + if err != nil { + panic(fmt.Sprintf("commit failed on bkey '%v': err '%v'", string(bkey), err)) + } + tx.tx = tx.Db.db.NewTransactionAt(1, tx.write) + + tx.writeCount = 1 + tx.writeByteCount = sz + } + err := tx.tx.SetEntry(entry) + + // ErrTxnTooBig is returned if too many writes are fit into a single transaction. + // badger docs: "An ErrTxnTooBig will be reported in case the number of pending + // writes/deletes in the transaction exceeds a certain limit. In that case, it + // is best to commit the transaction and start a new transaction immediately." + // + if err == badger.ErrTxnTooBig { + + err = tx.tx.CommitAt(1, nil) + if err != nil { + panic(fmt.Sprintf("commit after TooBig failed on bkey '%v': err '%v'", string(bkey), err)) + } + + tx.tx = tx.Db.db.NewTransactionAt(1, tx.write) + //vv("NewTx write txn (p=%p) on gid=%v. b/c TooBig writeCount=%v; writeByteCount=%v", tx.tx, curGID(), tx.writeCount, tx.writeByteCount) + tx.writeCount = 1 + tx.writeByteCount = sz + + err = tx.tx.SetEntry(entry) + panicOn(err) + //panic(fmt.Sprintf("got error badger.ErrTxnTooBig, but we shoud never get this now; len(by) = %v; len(bkey)=%v; vs limit is 10MB. tx.writeCount=%v; tx.writeByteCount=%v;", len(by), len(bkey), tx.writeCount, tx.writeByteCount)) + } + return err +} + +// RemoveContainer deletes the container specified by the shard and container key ckey +func (tx *BadgerTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error { + bkey := txkey.Key(index, field, view, shard, ckey) + tx.mu.Lock() + err := tx.tx.Delete(bkey) + tx.mu.Unlock() + return err +} + +// Add sets all the a bits hot in the specified fragment. +func (tx *BadgerTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) { + return tx.addOrRemove(index, field, view, shard, batched, false, a...) +} + +// Remove clears all the specified a bits in the chosen fragment. +func (tx *BadgerTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { + const batched = false + const remove = true + return tx.addOrRemove(index, field, view, shard, batched, remove, a...) +} + +func (tx *BadgerTx) addOrRemove(index, field, view string, shard uint64, batched, remove bool, a ...uint64) (changeCount int, err error) { + // pure hack to match RoaringTx + defer func() { + if !remove && !batched { + if changeCount > 0 { + changeCount = 1 + } + } + }() + + if len(a) == 0 { + return 0, nil + } + + // have to sort, b/c input is not always sorted. + sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) + + var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. + var rc *roaring.Container + var hi uint64 + var lo uint16 + + for i, v := range a { + + hi, lo = highbits(v), lowbits(v) + if hi != lastHi { + // either first time through, or changed to a different container. + // do we need put the last updated container now? + if i > 0 { + // not first time through, write what we got. + if remove && (rc == nil || rc.N() == 0) { + err = tx.RemoveContainer(index, field, view, shard, lastHi) + panicOn(err) + } else { + err = tx.PutContainer(index, field, view, shard, lastHi, rc) + panicOn(err) + } + } + // get the next container + rc, err = tx.Container(index, field, view, shard, hi) + panicOn(err) + } // else same container, keep adding bits to rct. + chng := false + // rc can be nil before, and nil after, in both Remove/Add below. + // The roaring container add() and remove() methods handle this. + if remove { + rc, chng = rc.Remove(lo) + } else { + rc, chng = rc.Add(lo) + } + if chng { + changeCount++ + } + lastHi = hi + } + // write the last updates. + if remove { + if rc == nil || rc.N() == 0 { + err = tx.RemoveContainer(index, field, view, shard, hi) + panicOn(err) + } else { + err = tx.PutContainer(index, field, view, shard, hi, rc) + panicOn(err) + } + } else { + if rc == nil || rc.N() == 0 { + panic("there should be no way to have an empty bitmap AFTER an Add() operation") + } + err = tx.PutContainer(index, field, view, shard, hi, rc) + panicOn(err) + } + return +} + +// Contains returns exists true iff the bit chosen by key is +// hot (set to 1) in specified fragment. +func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { + + lo, hi := lowbits(key), highbits(key) + bkey := txkey.Key(index, field, view, shard, hi) + tx.mu.Lock() + item, err := tx.tx.Get(bkey) + tx.mu.Unlock() + if err == badger.ErrKeyNotFound { + return false, nil + } + if err != nil { + return false, err + } + err = item.Value(func(v []byte) error { + // This func with val would only be called if item.Value encounters no error + c := tx.toContainer(item.UserMeta(), v) + exists = c.Contains(lo) + return nil + }) + return exists, err +} + +func (tx *BadgerTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { + + prefix := txkey.AllShardPrefix(index, field, view) + + bi := NewBadgerIterator(tx, prefix) + defer bi.Close() + bi.Seek(prefix) + if !bi.it.Valid() { + return + } + lastShard := uint64(0) + firstDone := false + for bi.Next() { + item := bi.it.Item() + key := item.Key() + shard := txkey.ShardFromKey(key) + if firstDone { + if shard != lastShard { + sliceOfShards = append(sliceOfShards, shard) + } + lastShard = shard + } else { + // first time + lastShard = shard + firstDone = true + sliceOfShards = append(sliceOfShards, shard) + } + + } + return +} + +// key is the container key for the first roaring Container +// roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will +// return the first container at or after key. found will be true if a +// container is found at key. +// +// BadgerTx notes: We auto-stop at the end of this shard, not going beyond. +func (tx *BadgerTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { + + // needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000" + needle := txkey.Key(index, field, view, shard, firstRoaringContainerKey) + + // prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@" + prefix := txkey.Prefix(index, field, view, shard) + + bi := NewBadgerIterator(tx, prefix) + bi.Seek(needle) + if !bi.it.Valid() { + return bi, false, nil + } + + if !bi.it.ValidForPrefix(prefix) { + return bi, false, nil + } + item := bi.it.Item() + // have to compare b/c badger might give us valid iterator + // that is past our needle if needle isn't present. + return bi, bytes.Equal(item.Key(), needle), nil +} + +// BadgerIterator is the iterator returned from a BadgerTx.ContainerIterator() call. +// It implements the roaring.ContainerIterator interface. +type BadgerIterator struct { + tx *BadgerTx + it *badger.Iterator + + prefix []byte + seekto []byte + + // seen counts how many Next() calls we have seen. + // It is used to match roaring.ContainerIterator semantics. + // Also useful for testing. + seen int +} + +// NewBadgerIterator creates an iterator on tx that will +// only return txkey.Keys that start with prefix. +func NewBadgerIterator(tx *BadgerTx, prefix []byte) (bi *BadgerIterator) { + + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = false // else by default, pre-fetches the 1st 100 values, which would be slow. + opts.Reverse = false + + tx.mu.Lock() + it := tx.tx.NewIterator(opts) + tx.mu.Unlock() + + bi = &BadgerIterator{ + tx: tx, + it: it, + prefix: prefix, + } + tx.Db.muOpenTxIt.Lock() + if tx.Db.openIt == nil { + tx.Db.openIt = make(map[*BadgerIterator]bool) + } + tx.Db.openIt[bi] = false // true for reverse, false for forward iteration. + tx.Db.muOpenTxIt.Unlock() + + bi.it.Seek(prefix) + return +} + +// NewBadgerReverseIterator makes a highest-to-lowest key iterator. +// Only keys that are prefixed with prefix will be returned. +// seekto tells where to start, and should be typically shard+1 +// to start at the end of shard. Really only used in Max() at the moment. +// After creating a reverse badger iterator it, we will call it.Seek(seekto). +func NewBadgerReverseIterator(tx *BadgerTx, prefix, seekto []byte) (bi *BadgerIterator) { + + tx.Db.muOpenTxIt.Lock() + defer tx.Db.muOpenTxIt.Unlock() + + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = false // else by default, pre-fetches the 1st 100 values, which would be slow. + opts.Reverse = true + opts.Prefix = prefix // possible storage IOPs optimization by badger + tx.mu.Lock() + it := tx.tx.NewIterator(opts) + tx.mu.Unlock() + + bi = &BadgerIterator{ + tx: tx, + it: it, + prefix: prefix, + seekto: seekto, + } + if tx.Db.openIt == nil { + tx.Db.openIt = make(map[*BadgerIterator]bool) + } + bi.tx.Db.openIt[bi] = true // true for reverse, false for forward iteration. + bi.it.Seek(seekto) + return +} + +// Close tells the database and transaction that the user is done +// with the iterator. +// From the badger docs: It is important to call this when you're done with iteration. +// else you will get an error on tx.Discard()/Commit(). +func (bi *BadgerIterator) Close() { + + bi.tx.Db.muOpenTxIt.Lock() + delete(bi.tx.Db.openIt, bi) + bi.it.Close() + + bi.tx.Db.muOpenTxIt.Unlock() +} + +// Valid returns false if there are no more values in the iterator's range. +func (bi *BadgerIterator) Valid() bool { + return bi.it.Valid() +} + +// Seek allows the iterator to start at needle instead of the global begining. +func (bi *BadgerIterator) Seek(needle []byte) { + bi.it.Seek(needle) +} + +// Next advances the iterator. +func (bi *BadgerIterator) Next() bool { + + // have to skip the first bi.it.Next() call because badger iterators point to the + // first value immediately, but Pilosa iterators must have Next() called + // on a fresh iterator to get the first value. + if bi.seen > 0 { + bi.it.Next() + } + bi.seen++ + return bi.it.ValidForPrefix(bi.prefix) // does the bi.it.Valid() inside and false if not valid always. +} + +// Value retrieves what is pointed at currently by the iterator. +func (bi *BadgerIterator) Value() (containerKey uint64, c *roaring.Container) { + if !bi.it.Valid() { + panic("bi.it not valid") + } + item := bi.it.Item() + if item == nil { + panic("item was nil") + } + key := item.Key() + containerKey = txkey.KeyExtractContainerKey(key) + + err := item.Value(func(v []byte) error { + c = bi.tx.toContainer(item.UserMeta(), v) + return nil + }) + panicOn(err) + return +} + +// badgerFinder implements roaring.IteratorFinder. +// It is used by BadgerTx.ForEach() +type badgerFinder struct { + tx *BadgerTx + index string + field string + view string + shard uint64 + needClose []Closer +} + +// FindIterator lets badgerFinder implement the roaring.FindIterator interface. +func (bf *badgerFinder) FindIterator(seek uint64) (roaring.ContainerIterator, bool) { + a, found, err := bf.tx.ContainerIterator(bf.index, bf.field, bf.view, bf.shard, seek) + panicOn(err) + bf.needClose = append(bf.needClose, a) + return a, found +} + +// Close closes all bf.needClose listed Closers. +func (bf *badgerFinder) Close() { + for _, i := range bf.needClose { + i.Close() + } +} + +// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE +// the transaction Commits or Rollsback. +func (tx *BadgerTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { + bf := &badgerFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} + itr := roaring.NewIterator(bf) + return itr +} + +// ForEach applies fn to each bitmap in the fragment. +func (tx *BadgerTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { + itr := tx.NewTxIterator(index, field, view, shard) + defer itr.Close() + + // Seek can create many container iterators, thus bf.Close() needClose list. + itr.Seek(0) + // v is the bit we are operating on. + for v, eof := itr.Next(); !eof; v, eof = itr.Next() { + if err := fn(v); err != nil { + return err + } + } + return nil +} + +// ForEachRange applies fn on the selected range of bits on the chosen fragment. +func (tx *BadgerTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { + + itr := tx.NewTxIterator(index, field, view, shard) + defer itr.Close() + + itr.Seek(start) + + // v is the bit we are operating on. + for v, eof := itr.Next(); !eof && v < end; v, eof = itr.Next() { + if err := fn(v); err != nil { + return err + } + } + return nil +} + +// Count operates on the full bitmap level, so it sums over all the containers +// in the bitmap. +func (tx *BadgerTx) Count(index, field, view string, shard uint64) (uint64, error) { + + a, found, err := tx.ContainerIterator(index, field, view, shard, 0) + panicOn(err) + defer a.Close() + if !found { + return 0, nil + } + result := int32(0) + for a.Next() { + ckey, cont := a.Value() + _ = ckey + result += cont.N() + } + + return uint64(result), nil +} + +// Max is the maximum bit-value in your bitmap. +// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does. +func (tx *BadgerTx) Max(index, field, view string, shard uint64) (uint64, error) { + + prefix := txkey.Prefix(index, field, view, shard) + seekto := txkey.Prefix(index, field, view, shard+1) + + it := NewBadgerReverseIterator(tx, prefix, seekto) // this iterator is still open, when we commit/discard tx. + defer it.Close() + + if !it.it.Valid() { + return 0, nil + } + hb, rc := it.Value() // getting it returns invalid, as in empty iterator + lb := rc.Max() + + return hb<<16 | uint64(lb), nil +} + +// Min returns the smallest bit set in the fragment. If no bit is hot, +// the second return argument is false. +func (tx *BadgerTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { + + // Seek can create many container iterators, thus the bf.Close() needClose list. + bf := &badgerFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} + defer bf.Close() + itr := roaring.NewIterator(bf) + + itr.Seek(0) + + // v is the bit we are operating on. + v, eof := itr.Next() + if eof { + return 0, false, nil + } + return v, true, nil +} + +// UnionInPlace unions all the others Bitmaps into a new Bitmap, and then writes it to the +// specified fragment. +func (tx *BadgerTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { + + rbm, err := tx.RoaringBitmap(index, field, view, shard) + panicOn(err) + + rbm.UnionInPlace(others...) + // iterate over the containers that changed within rbm, and write them back to disk. + + it, found := rbm.Containers.Iterator(0) + _ = found // don't care about the value of found, because first containerKey might be > 0 + + for it.Next() { + containerKey, rc := it.Value() + + // TODO: only write the changed ones back, as optimization? + // Compare to ImportRoaringBits. + err := tx.PutContainer(index, field, view, shard, containerKey, rc) + panicOn(err) + } + return nil +} + +// CountRange returns the count of hot bits in the start, end range on the fragment. +// roaring.countRange counts the number of bits set between [start, end). +func (tx *BadgerTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { + + if tx.frag == nil { + return tx.countRangeNoFrag(index, field, view, shard, start, end) + } + + // For speed, exploit the fact that on startup the rowCache will + // have already loaded fragments. + rowID := start / ShardWidth + row, err := tx.frag.unprotectedRow(tx, rowID) + if err != nil { + return 0, err + } + return row.Count(), nil +} + +// CountRange returns the count of hot bits in the start, end range on the fragment. +// roaring.countRange counts the number of bits set between [start, end). +func (tx *BadgerTx) countRangeNoFrag(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { + + if start >= end { + return 0, nil + } + + skey := highbits(start) + ekey := highbits(end) + + citer, found, err := tx.ContainerIterator(index, field, view, shard, skey) + _ = found + panicOn(err) + + defer citer.Close() + + // If range is entirely in one container then just count that range. + if skey == ekey { + citer.Next() + _, c := citer.Value() + return uint64(c.CountRange(int32(lowbits(start)), int32(lowbits(end)))), nil + } + + for citer.Next() { + k, c := citer.Value() + if k < skey { + citer.Close() + panic(fmt.Sprintf("should be impossible for k(%v) to be less than skey(%v). tx p=%p", k, skey, tx)) + } + + // k > ekey handles the case when start > end and where start and end + // are in different containers. Same container case is already handled above. + if k > ekey { + break + } + if k == skey { + n += uint64(c.CountRange(int32(lowbits(start)), roaring.MaxContainerVal+1)) + continue + } + if k < ekey { + n += uint64(c.N()) + continue + } + if k == ekey { + n += uint64(c.CountRange(0, int32(lowbits(end)))) + break + } + } + + return n, nil +} + +// OffsetRange creates a new roaring.Bitmap to return in other. For all the +// hot bits in [start, endx) of the chosen fragment, it stores +// them into other but with offset added to their bit position. +// The primary client is doing this, using ShardWidth, already; see +// fragment.rowFromStorage() in fragment.go. For example: +// +// data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard, +// f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) +// ^ offset ^ start ^ endx +// +// The start and endx arguments are container keys that have been shifted left by 16 bits; +// their highbits() will be taken to determine the actual container keys. This +// is done to conform to the roaring.OffsetRange() argument convention. +// +func (tx *BadgerTx) OffsetRange(index, field, view string, shard, offset, start, endx uint64) (other *roaring.Bitmap, err error) { + + // roaring does these three checks in its OffsetRange + if lowbits(offset) != 0 { + panic("offset must not contain low bits") + } + if lowbits(start) != 0 { + panic("range start must not contain low bits") + } + if lowbits(endx) != 0 { + panic("range end must not contain low bits") + } + + other = roaring.NewSliceBitmap() + off := highbits(offset) + hi0, hi1 := highbits(start), highbits(endx) + + needle := txkey.Key(index, field, view, shard, hi0) + prefix := txkey.Prefix(index, field, view, shard) + + n2, pre2 := txkey.KeyAndPrefix(index, field, view, shard, hi0) + if string(n2) != string(needle) { + panic(fmt.Sprintf("problem! n2(%v) != needle(%v), txkey.KeyAndPrefix not consitent with txkey.Key()", string(n2), string(needle))) + } + if string(pre2) != string(prefix) { + panic(fmt.Sprintf("problem! pre2(%v) != prefix(%v), txkey.KeyAndPrefix not consitent with txkey.Key()", string(pre2), string(prefix))) + } + + it := NewBadgerIterator(tx, prefix) + defer it.Close() + it.Seek(needle) + for ; it.it.ValidForPrefix(prefix); it.Next() { + item := it.it.Item() + bkey := item.Key() + k := txkey.KeyExtractContainerKey(bkey) + + // >= hi1 is correct b/c endx cannot have any lowbits set. + if uint64(k) >= hi1 { + break + } + destCkey := off + (k - hi0) + err := item.Value(func(v []byte) error { + c := tx.toContainer(item.UserMeta(), v) + other.Containers.Put(destCkey, c.Freeze()) + + return nil + }) + if err != nil { + return nil, err + } + } + return other, nil +} + +// IncrementOpN increments the tx opcount by changedN +func (tx *BadgerTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { + tx.opcount += changedN +} + +// ImportRoaringBits handles deletes by setting clear=true. +// rowSet[rowID] returns the number of bit changed on that rowID. +func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { + n := itr.Len() + if n == 0 { + return + } + rowSet = make(map[uint64]int) + + var currRow uint64 + + var oldC *roaring.Container + for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { + if rowSize != 0 { + currRow = itrKey / rowSize + } + nsynth := int(synthC.N()) + if nsynth == 0 { + continue + } + // INVAR: nsynth > 0 + + oldC, err = tx.Container(index, field, view, shard, itrKey) + panicOn(err) + if err != nil { + return + } + + if oldC == nil || oldC.N() == 0 { + // no container at the itrKey in badger (or all zero container). + if clear { + // changed of 0 and empty rowSet is perfect, no need to change the defaults. + continue + } else { + + changed += nsynth + rowSet[currRow] += nsynth + + err = tx.PutContainer(index, field, view, shard, itrKey, synthC) + if err != nil { + return + } + continue + } + } + + if clear { + existN := oldC.N() // number of bits set in the old container + newC := oldC.Difference(synthC) + + // update rowSet and changes + if newC.N() == existN { + // INVAR: do changed need adjusting? nope. same bit count, + // so no change could have happened. + continue + } else { + changes := int(existN - newC.N()) + changed += changes + rowSet[currRow] -= changes + + if tx.DeleteEmptyContainer && newC.N() == 0 { + err = tx.RemoveContainer(index, field, view, shard, itrKey) + if err != nil { + return + } + continue + } + err = tx.PutContainer(index, field, view, shard, itrKey, newC) + if err != nil { + return + } + continue + } + } else { + // setting bits + + existN := oldC.N() + if existN == roaring.MaxContainerVal+1 { + // completely full container already, set will do nothing. so changed of 0 default is perfect. + continue + } + if existN == 0 { + // can nsynth be zero? No, because of the continue/invariant above where nsynth > 0 + changed += nsynth + rowSet[currRow] += nsynth + err = tx.PutContainer(index, field, view, shard, itrKey, synthC) + if err != nil { + return + } + continue + } + + newC := roaring.Union(oldC, synthC) // UnionInPlace was giving us crashes on overly large containers. + + if roaring.ContainerType(newC) == roaring.ContainerBitmap { + newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it. + } + if newC.N() != existN { + changes := int(newC.N() - existN) + changed += changes + rowSet[currRow] += changes + + err = tx.PutContainer(index, field, view, shard, itrKey, newC) + if err != nil { + panicOn(err) + return + } + continue + } + } + } + return +} + +func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) { + + if len(v) == 0 { + return nil + } + + var w []byte + useRowCache := tx.UseRowCache() + if tx.doAllocZero || useRowCache { + // Do electric fence-inspired bad-memory read detection. + // + // The v []byte lives in BadgerDB's memory-mapped vlog-file, + // and Badger will recycle it after tx ends with rollback or commit. + // + // Problem is, at least some operations were not respecting transaction boundaries. + // This technique helped us find them. The rowCache was an example. + // + // See the global const DetectMemAccessPastTx + // at the top of txfactory.go to activate/deactivate this. + // + // Seebs suggested this nice variation: we could use individual mmaps for these + // copies, which would be unusable in production, but workable for testing, and then unmap them, + // which would get us probable segfaults on future accesses to them. + // + // The go runtime also has an -efence flag which may be similarly useful if really pressed. + // + w = make([]byte, len(v)) + copy(w, v) + + if !useRowCache { + // register w so we can catch out-of-tx memory access + tx.acMu.Lock() + defer tx.acMu.Unlock() + tx.ourAllocs = append(tx.ourAllocs, w) + } + } else { + w = v + } + + switch typ { + case roaring.ContainerArray: + c := roaring.NewContainerArray(toArray16(w)) + if tx.doAllocZero { + // tx.acMu was acquired above, and Unlock deferred. + tx.ourContainers = append(tx.ourContainers, c) + } + return c + case roaring.ContainerBitmap: + c := roaring.NewContainerBitmap(-1, toArray64(w)) + if tx.doAllocZero { + // tx.acMu was acquired above, and Unlock deferred. + tx.ourContainers = append(tx.ourContainers, c) + } + return c + case roaring.ContainerRun: + c := roaring.NewContainerRun(toInterval16(w)) + if tx.doAllocZero { + // tx.acMu was acquired above, and Unlock deferred. + tx.ourContainers = append(tx.ourContainers, c) + } + return c + default: + panic(fmt.Sprintf("unknown container: %v", typ)) + } +} + +// StringifiedBadgerKeys returns a string with all the container +// keys available in badger. +func (w *BadgerDBWrapper) StringifiedBadgerKeys(optionalUseThisTx Tx, short bool) (r string) { + if optionalUseThisTx == nil { + tx, _ := w.NewTx(!writable, "", Txo{}) + defer tx.Rollback() + r = stringifiedBadgerKeysTx(tx.(*BadgerTx), short) + return + } + + btx, ok := optionalUseThisTx.(*BadgerTx) + if !ok { + return fmt.Sprintf("", optionalUseThisTx) + } + r = stringifiedBadgerKeysTx(btx, short) + return +} + +// countBitsSet returns the number of bits set (or "hot") in +// the roaring container value found by the txkey.Key() +// formatted bkey. +func (tx *BadgerTx) countBitsSet(bkey []byte) (n int) { + + item, err := tx.tx.Get(bkey) + if err == badger.ErrKeyNotFound { + panic(fmt.Sprintf("badger did not have value for bkey = '%v'", string(bkey))) + } + panicOn(err) + + var rc *roaring.Container + err = item.Value(func(v []byte) error { + // This func with val would only be called if item.Value encounters no error + rc = tx.toContainer(item.UserMeta(), v) + return nil + }) + panicOn(err) + + n = int(rc.N()) + return +} + +func (tx *BadgerTx) Dump(short bool) { + fmt.Printf("BadgerTx %p Dump: %v\n", tx, stringifiedBadgerKeysTx(tx, short)) +} + +// stringifiedBadgerKeysTx reports all the badger keys and a +// corresponding blake3 hash viewable by txn within the entire +// badger database. +// It also reports how many bits are hot in the roaring container +// (how many bits are set, or 1 rather than 0). +// +// By convention, we must return the empty string if there +// are no keys present. The tests use this to confirm +// an empty database. +func stringifiedBadgerKeysTx(tx *BadgerTx, short bool) (r string) { + + r = "allkeys:[\n" + it := tx.tx.NewIterator(badger.DefaultIteratorOptions) // PrefetchValues true okay here. + defer it.Close() + any := false + for it.Rewind(); it.Valid(); it.Next() { + any = true + item := it.Item() + bkey := item.Key() + key := txkey.ToString(bkey) + ckey := txkey.KeyExtractContainerKey(bkey) + h := "" + srbm := "" + err := item.Value(func(val []byte) error { + h = hash.Blake3sum16(val) + ct := tx.toContainer(item.UserMeta(), val) + cts := roaring.NewSliceContainers() + cts.Put(ckey, ct) + rbm := &roaring.Bitmap{Containers: cts} + srbm = BitmapAsString(rbm) + return nil + }) + panicOn(err) + r += fmt.Sprintf("%v -> %v (%v hot)\n", key, h, tx.countBitsSet(bkey)) + if !short { + r += " ......." + srbm + "\n" + } + } + r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + + if !any { + return "" + } + return "badger-" + r +} + +func (w *BadgerDBWrapper) DeleteField(index, field, fieldPath string) error { + + // under blue-green roaring_badger, the directory will not be found, b/c roaring will have + // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: + // "If the path does not exist, RemoveAll returns nil (no error)" + err := os.RemoveAll(fieldPath) + if err != nil { + return errors.Wrap(err, "removing directory") + } + prefix := txkey.FieldPrefix(index, field) + return w.DeletePrefix(prefix) +} + +func (w *BadgerDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { + prefix := txkey.Prefix(index, field, view, shard) + return w.DeletePrefix(prefix) +} + +func (w *BadgerDBWrapper) DeletePrefix(prefix []byte) error { + w.muDb.Lock() + defer w.muDb.Unlock() + + // a) do key-ony iteration, no value fetch; + // + // b) do deletes in large batches, to avoid alot of txn overhead; + // per recommendation https://github.com/dgraph-io/badger/issues/598 + // + // c) we do not, at present, try to maintain one large + // transaction with all the keys in a index in it. Because + // there can be too many keys. Hence the index will disappear + // in chucks of 100K keys, not atomically-all-at-once. + + noMoreKeysWithPrefix := false + const maxDeletesPerTxn = 100000 + + for !noMoreKeysWithPrefix { + + txn := w.db.NewTransactionAt(1, true) + //defer txn.Discard() + + o := badger.DefaultIteratorOptions + o.AllVersions = false + o.PrefetchValues = false // key-only iteration, no values. + + // note: panic: Unclosed iterator at time of Txn.Discard ? panic on segfault here? + // This means we messed up and Closed() the Database already; too early. For + // example in TxFactor.CloseIndex() in txfactory.go:331. + it := txn.NewIterator(o) + + //defer it.Close() + n := 0 + goners := make([][]byte, 0, maxDeletesPerTxn) + for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() { + + // KeyCopy() is required; Key() means corruption and possible segfault. + key := it.Item().KeyCopy(nil) + goners = append(goners, key) + n++ + if n >= maxDeletesPerTxn { + break + } + } + if !it.ValidForPrefix(prefix) { + noMoreKeysWithPrefix = true // done with the full delete of up to maxDeletesPerTxn + } + for _, key := range goners { + if err := txn.Delete(key); err != nil { + it.Close() + txn.Discard() + return err + } + } + it.Close() + err := txn.CommitAt(1, nil) + + // err back from Update can be ErrConflict in case of + // a conflict. Badger docs: "Depending on the state + // of your application, you have the option to + // retry the operation if you receive this error." + panicOn(err) + + } // end for: proceed to next bath of 100K keys + + // Finally, run a garbage collection to delete values from the value log. + // + // "Only one GC is allowed at a time. If another value log GC + // is running, or DB has been closed, this would return an ErrRejected." + // -- https://godoc.org/github.com/dgraph-io/badger#DB.RunValueLogGC + // Still, we don't see a mutex inside the RunValueLogGC code, so + // lock muGC just to be sure. + w.muGC.Lock() + defer w.muGC.Unlock() + _ = w.db.RunValueLogGC(0.5) + + return nil +} + +func (tx *BadgerTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { + + rbm, err := tx.RoaringBitmap(index, field, view, shard) + if err != nil { + return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") + } + var buf bytes.Buffer + sz, err = rbm.WriteTo(&buf) + if err != nil { + return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") + } + return ioutil.NopCloser(&buf), sz, err +} + +func (w *BadgerDBWrapper) OpenListString() (r string) { + return "BadgerDBWrapper.OpenListString not yet implemented" +} +func (w *BadgerDBWrapper) OpenSnList() (sns []int64) { + return nil // panic("BadgerDBWrapper.OpenSnList not yet implemented") +} + +// Sn retreives the serial number of the Tx. +func (tx *BadgerTx) Sn() int64 { + return tx.sn +} + +func (w *BadgerDBWrapper) Update(fn func(txn *badger.Txn) error) error { + txn := w.db.NewTransactionAt(1, true) + defer txn.Discard() + + if err := fn(txn); err != nil { + return err + } + + return txn.CommitAt(1, nil) +} diff --git a/badger_test.go b/badger_test.go new file mode 100644 index 000000000..a4ad6ab71 --- /dev/null +++ b/badger_test.go @@ -0,0 +1,1756 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// explanation of build tags: +// +// badgerdb builds but won't run in 32-bit 386 world, as of 2020 July 20. +// See https://github.com/dgraph-io/badger/issues/1384 for any progress. +// What we see is that the value-log allocations immediately run out of +// memory. So we turn off 386 with a build tag to keep the .circleci happy. + +// +build !386 + +package pilosa + +import ( + "fmt" + "math" + "os" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/pilosa/pilosa/v2/roaring" + "github.com/pilosa/pilosa/v2/testhook" + "github.com/pkg/errors" +) + +var _ = &roaring.Bitmap{} + +func init() { + testhook.RegisterPostTestHook(reportTestBadgersNeedingClose) +} + +// helpers, each runs their own new txn, and commits if a change/delete +// was made. The txn is rolled back if it is just viewing the data. + +func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) { + + tx, _ := dbwrap.NewTx(!writable, index, Txo{}) + defer tx.Rollback() + exists, err := tx.Contains(index, field, view, shard, bitvalue) + panicOn(err) + if !exists { + panic(fmt.Sprintf("ARG bitvalue '%v' was NOT SET!!!", bitvalue)) + } + + tx.Rollback() +} + +func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) { + + tx, _ := dbwrap.NewTx(!writable, index, Txo{}) + defer tx.Rollback() + exists, err := tx.Contains(index, field, view, shard, bitvalue) + panicOn(err) + if exists { + panic(fmt.Sprintf("ARG bitvalue '%v' WAS SET but should not have been.!!!", bitvalue)) + } + tx.Rollback() +} + +func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) { + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + + // add a bit + changed, err := tx.Add(index, field, view, shard, doBatched, putme) + if changed != 1 { + panic("should have 1 bit changed") + } + panicOn(err) + + exists, err := tx.Contains(index, field, view, shard, putme) + panicOn(err) + if !exists { + panic("ARG putme was NOT SET!!!") + } + panicOn(tx.Commit()) +} + +func badgerDBMustDeleteBitvalueContainer(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) { + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + hi := highbits(putme) + panicOn(tx.RemoveContainer(index, field, view, shard, hi)) + panicOn(tx.Commit()) +} + +func badgerDBMustDeleteBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) { + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + _, err := tx.Remove(index, field, view, shard, putme) + panicOn(err) + panicOn(tx.Commit()) +} + +func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func()) { + var err error + fn := badgerPath(path) + panicOn(os.RemoveAll(fn)) + ww, err := globalBadgerReg.OpenDBWrapper(path, DetectMemAccessPastTx) + panicOn(err) + w = ww.(*BadgerDBWrapper) + + // verify it is empty + allkeys := w.StringifiedBadgerKeys(nil, true) + if allkeys != "" { + panic(fmt.Sprintf("freshly created database was not empty! had keys:'%v'", allkeys)) + } + + return w, func() { + w.Close() // stop any started background GC goroutine. + os.RemoveAll(fn) + } +} + +// end of helper utilities +////////////////////////// + +////////////////////////// +// begin Tx method tests + +func TestBadger_DeleteFragment(t *testing.T) { + + // setup + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteFragment") + defer clean() + defer dbwrap.Close() + index, field, view, shard0 := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + + shard1 := uint64(1) + + bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} + shards := []uint64{shard0, shard1} + for _, s := range shards { + for _, v := range bits { + changed, err := tx.Add(index, field, view, s, doBatched, v) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + } + } + + for _, s := range shards { + for _, v := range bits { + exists, err := tx.Contains(index, field, view, s, v) + panicOn(err) + if !exists { + panic("ARG bitvalue was NOT SET!!!") + } + } + } + err := tx.Commit() + panicOn(err) + + // end of setup + + survivor := shard0 + victim := shard1 + err = dbwrap.DeleteFragment(index, field, view, victim, nil) + panicOn(err) + + tx, _ = dbwrap.NewTx(!writable, index, Txo{}) + defer tx.Rollback() + + for _, s := range shards { + for _, v := range bits { + exists, err := tx.Contains(index, field, view, s, v) + panicOn(err) + if s == survivor { + if !exists { + panic(fmt.Sprintf("ARG survivor died : bit %v", v)) + } + } else if s == victim { // victim, should have been deleted + if exists { + panic(fmt.Sprintf("ARG victim lived : bit %v", v)) + } + } + } + } +} + +func TestBadger_Max_on_many_containers(t *testing.T) { + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Max_on_many_containers") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + + putmeValues := []uint64{0, 2 << 16, 4 << 16} + + for _, putme := range putmeValues { + badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + } + + tx, _ := dbwrap.NewTx(!writable, index, Txo{}) + defer tx.Rollback() + + max, err := tx.Max(index, field, view, shard) + panicOn(err) + expected := putmeValues[len(putmeValues)-1] + if max != expected { + panic(fmt.Sprintf("expected Max() of %v but got max=%v", expected, max)) + } +} + +// and the rest + +func TestBadger_SetBitmap(t *testing.T) { + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SetBitmap") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + bitvalue := uint64(0) + changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + + exists, err := tx.Contains(index, field, view, shard, bitvalue) + panicOn(err) + if !exists { + panic("ARG bitvalue was NOT SET!!!") + } + + err = tx.Commit() + panicOn(err) + + // + // commited, so should be visible outside the txn + // + + tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) + exists, err = tx2.Contains(index, field, view, shard, bitvalue) + panicOn(err) + if !exists { + panic("ARG bitvalue was NOT SET!!! on tx2") + } + + n, err := tx2.Count(index, field, view, shard) + panicOn(err) + if n != 1 { + panic(fmt.Sprintf("should have Count 1; instead n = %v", n)) + } + tx2.Rollback() +} + +func TestBadger_OffsetRange(t *testing.T) { + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_OffsetRange") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + + bitvalue := uint64(1 << 20) + changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + + bitvalue2 := uint64(1<<20 + 1) + changed, err = tx.Add(index, field, view, shard, doBatched, bitvalue2) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + + exists, err := tx.Contains(index, field, view, shard, bitvalue) + panicOn(err) + if !exists { + panic("ARG bitvalue was NOT SET!!!") + } + exists, err = tx.Contains(index, field, view, shard, bitvalue2) + panicOn(err) + if !exists { + panic("ARG bitvalue2 was NOT SET!!!") + } + + err = tx.Commit() + panicOn(err) + + offset := uint64(0 << 20) + start := uint64(0 << 16) + endx := bitvalue + 1<<16 + + tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) + rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx) + panicOn(err) + tx2.Rollback() + + // should see our 1M value + s2 := BitmapAsString(rbm2) + expect2 := "c(1048576, 1048577)" + if s2 != expect2 { + panic(fmt.Sprintf("s2='%v', but expected '%v'", s2, expect2)) + } + + // now offset by 2M + offset = uint64(2 << 20) + tx3, _ := dbwrap.NewTx(!writable, index, Txo{}) + rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx) + panicOn(err) + tx3.Rollback() + + //expect to see 3M == 3145728 + s3 := BitmapAsString(rbm3) + expect3 := "c(3145728, 3145729)" + + if s3 != expect3 { + panic(fmt.Sprintf("s3='%v', but expected '%v'", s3, expect3)) + } +} + +func TestBadger_Count_on_many_containers(t *testing.T) { + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Count_on_many_containers") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + + putmeValues := []uint64{0, 2 << 16, 4 << 16} + + for _, putme := range putmeValues { + badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + } + + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + defer tx.Rollback() + + n, err := tx.Count(index, field, view, shard) + panicOn(err) + if int(n) != len(putmeValues) { + panic(fmt.Sprintf("expected Count of %v but got n=%v", len(putmeValues), n)) + } +} + +func TestBadger_Count_dense_containers(t *testing.T) { + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Count_dense_containers") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + + expected := 0 + // can't do more than about 100k writes per badger txn by default, so + // have to keep this kind of small. + // (See maxBatchCount:104857, maxBatchSize:10066329). + for i := uint64(0); i < (1<<16)+2; i += 2 { + changed, err := tx.Add(index, field, view, shard, doBatched, i) + panicOn(err) + if changed <= 0 { + panic("wat? should have changed") + } + expected++ + } + defer tx.Rollback() + + n, err := tx.Count(index, field, view, shard) + panicOn(err) + if int(n) != expected { + panic(fmt.Sprintf("expected Count of %v but got n=%v", expected, n)) + } +} + +func TestBadger_ContainerIterator_on_empty(t *testing.T) { + // iterate on empty container, should not find anything. + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(!writable, index, Txo{}) + defer tx.Rollback() + bitvalue := uint64(0) + citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue) + panicOn(err) + defer citer.Close() + if found { + panic("should not have found anything") + } + panicOn(err) +} + +func TestBadger_ContainerIterator_on_one_bit(t *testing.T) { + // set one bit, iterate. + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + defer tx.Rollback() + + bitvalue := uint64(42) + + // add a bit + changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + + exists, err := tx.Contains(index, field, view, shard, bitvalue) + panicOn(err) + if !exists { + panic("ARG bitvalue was NOT SET!!!") + } + + // same Tx, continues in use. + + citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(bitvalue)) + if !found { + panic("ContainerIterator did not find the 42 bit") + } + panicOn(err) + defer citer.Close() + + loopCount := 0 + for citer.Next() { + key, container := citer.Value() + if key != 0 { + panic("42 should have had key 0") + } + if container == nil { + panic("container was nil") + } + if container.N() != 1 { + panic("put a bit in, but size of container was not 1") + } + if !container.Contains(lowbits(bitvalue)) { + panic("container did not have our bitvalue!") + } + loopCount++ + if loopCount > 0 { // happier linter + break + } + } + if loopCount != 1 { + panic("ContainerIterator did not return a citer that scanned our set bit") + } +} + +func TestBadger_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) { + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + defer tx.Rollback() + + putme := uint64(1<<16) + 3 // in the key:1 container + searchme := putme + 1 + + // add a bit + changed, err := tx.Add(index, field, view, shard, doBatched, putme) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + + exists, err := tx.Contains(index, field, view, shard, putme) + panicOn(err) + if !exists { + panic("ARG putme was NOT SET!!!") + } + + // same Tx, continues in use. + + citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(searchme)) + if !found { + panic("ContainerIterator did not find the searchme") + } + defer citer.Close() + loopCount := 0 + for citer.Next() { + key, container := citer.Value() + if key != 1 { + panic("Containeriterator searching for highbits(searchme) should not have had a bit") + } + if container == nil { + panic("container was nil") + } + if container.N() != 1 { + panic("put a bit in, but size of container was not 1") + } + if container.Contains(lowbits(searchme)) { + panic("container should have putme but not our searchme!") + } + loopCount++ + // only want first pass. keep linter happy by avoiding raw break + if loopCount > 0 { + break + } + } + panicOn(err) +} + +func TestBadger_ContainerIterator_empty_iteration_loop(t *testing.T) { + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_empty_iteration_loop") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + defer tx.Rollback() + + putme := uint64(1<<16) + 3 // in the key:1 container + searchme := uint64(1 << 17) // in the next container, key:2 + + // add a bit + changed, err := tx.Add(index, field, view, shard, doBatched, putme) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + + exists, err := tx.Contains(index, field, view, shard, putme) + panicOn(err) + if !exists { + panic("ARG putme was NOT SET!!!") + } + + // same Tx, continues in use. + + citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(searchme)) + panicOn(err) + if found { + panic("ContainerIterator found the searchme, when it should not have") + } + defer citer.Close() + if citer.Next() { + panic("expected no looping, 0 iterations, b/c started searchme past our data in putme") + } + + // expect to see a blow up from the citer.Value() call, verify that we do. + func() { + defer func() { + r := recover() + if r == nil { + panic("expected a panic from citer.Value() in this case") + } + }() + citer.Value() // should panic + }() + +} + +func TestBadger_ForEach_on_one_bit(t *testing.T) { + + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + defer tx.Rollback() + + bitvalue := uint64(42) + + // add a bit + changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + + exists, err := tx.Contains(index, field, view, shard, bitvalue) + panicOn(err) + if !exists { + panic("ARG bitvalue was NOT SET!!!") + } + + // same Tx, continues in use. + count := 0 + err = tx.ForEach(index, field, view, shard, func(v uint64) error { + if v != bitvalue { + panic(fmt.Sprintf("bitvalue corrupt got %v want %v", v, bitvalue)) + } + count += 1 + return nil + }) + panicOn(err) + if count != 1 { + panic(fmt.Sprintf("Expected single iteration got %v ", count)) + } +} + +func TestBadger_RemoveContainer_one_bit_test(t *testing.T) { + + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_RemoveContainer_one_bit_test") + defer clean() + defer dbwrap.Close() + + index, field, view, shard := "i", "f", "v", uint64(0) + + putmeValues := []uint64{0, 13, 77, 1511} + + for _, putme := range putmeValues { + + // a) delete of whole container in a seperate txn. Commit should establish the deletion. + + badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) + badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + + // b) deletion + rollback on the txn should restore the deleted bit + + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + + // delete, but rollback instead of commit + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + hi := highbits(putme) + panicOn(tx.RemoveContainer(index, field, view, shard, hi)) + tx.Rollback() + + // verify that the rollback undid the deletion. + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + + // c) within one Tx, after delete it should be gone as viewed within the txn. + tx, _ = dbwrap.NewTx(writable, index, Txo{}) + hi = highbits(putme) + + exists, err := tx.Contains(index, field, view, shard, putme) + panicOn(err) + if !exists { + panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme)) + } + + panicOn(tx.RemoveContainer(index, field, view, shard, hi)) + + exists, err = tx.Contains(index, field, view, shard, putme) + panicOn(err) + if exists { + panic(fmt.Sprintf("ARG putme '%v' was SET even after RemoveContiner in this txn.", putme)) + } + + tx.Rollback() + + // verify that the rollback undid the deletion. + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + // leave with clean slate + badgerDBMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) + } +} + +func TestBadger_Remove_one_bit_test(t *testing.T) { + + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Remove_one_bit_test") + defer clean() + defer dbwrap.Close() + + index, field, view, shard := "i", "f", "v", uint64(0) + + putmeValues := []uint64{0, 13, 77, 1511} + + for _, putme := range putmeValues { + + // a) delete of whole container in a seperate txn. Commit should establish the deletion. + + badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustDeleteBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + + // b) deletion + rollback on the txn should restore the deleted bit + + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + + // delete, but rollback instead of commit + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + hi, lo := highbits(putme), lowbits(putme) + _, _ = hi, lo + _, err := tx.Remove(index, field, view, shard, hi) + panicOn(err) + tx.Rollback() + + // verify that the rollback undid the deletion. + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + + // c) within one Tx, after delete it should be gone as viewed within the txn. + tx, _ = dbwrap.NewTx(writable, index, Txo{}) + + exists, err := tx.Contains(index, field, view, shard, putme) + panicOn(err) + if !exists { + panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme)) + } + + mustRemove(tx.Remove(index, field, view, shard, putme)) + + exists, err = tx.Contains(index, field, view, shard, putme) + panicOn(err) + if exists { + panic(fmt.Sprintf("ARG putme '%v' was SET even after Remove in this txn.", putme)) + } + + tx.Rollback() + + // verify that the rollback undid the deletion. + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + // leave with clean slate + badgerDBMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) + } +} + +func TestBadger_reverse_badger_iterator(t *testing.T) { + + // sanity check our understanding of Seek()-ing on reverse iterators. + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_reverse_badger_iterator") + defer clean() + defer dbwrap.Close() + + // add 0, 1, 2 to badgerdb as keys (and same value). + err := dbwrap.Update(func(txn *badger.Txn) error { + for i := 0; i < 3; i++ { + kv := []byte(fmt.Sprintf("a:%v", i)) + err := txn.Set(kv, kv) + panicOn(err) + } + return nil + }) + panicOn(err) + + tx := dbwrap.db.NewTransactionAt(1, !writable) + + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = false // else by default, pre-fetches the 1st 100 values, which would be slow. + opts.Reverse = true + it := tx.NewIterator(opts) + it.Rewind() + if !it.Valid() { + panic("invalid reversed iterator?") + } + a := []byte("a:3") + it.Seek(a) + if !it.Valid() { + panic("invalid reversed iterator after seek") + } + + it.Next() + if !it.Valid() { + panic("invalid reversed iterator after seek and next") + } + item := it.Item() + + err = item.Value(func(val []byte) error { + // This func with val would only be called if item.Value encounters no error. + if string(val) != "a:1" { + panic(fmt.Sprintf("we are in trouble, should have gotten 'a:1' but instead got '%v'", string(val))) + } + return nil + }) + panicOn(err) +} + +func TestBadger_Min_on_many_containers(t *testing.T) { + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Min_on_many_containers") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + + // verify no containers flag works + tx, _ := dbwrap.NewTx(!writable, index, Txo{}) + min, containersExist, err := tx.Min(index, field, view, shard) + _ = min + panicOn(err) + if containersExist { + panic("no containers should exist") + } + tx.Rollback() + + putmeValues := []uint64{3, 2 << 16, 4 << 16} + + for _, putme := range putmeValues { + badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + } + + tx, _ = dbwrap.NewTx(!writable, index, Txo{}) + defer tx.Rollback() + + min, containersExist, err = tx.Min(index, field, view, shard) + panicOn(err) + if !containersExist { + panic("containers should exist") + } + expected := putmeValues[0] + if min != expected { + panic(fmt.Sprintf("expected Min() of %v but got min=%v", expected, min)) + } +} + +func TestBadger_CountRange_on_many_containers(t *testing.T) { + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_CountRange_on_many_containers") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + + // verify no containers flag works + tx, _ := dbwrap.NewTx(!writable, index, Txo{}) + n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64) + panicOn(err) + if n != 0 { + panic("no containers should exist") + } + tx.Rollback() + + putmeValues := []uint64{3, 2 << 16, 4 << 16} + + for _, putme := range putmeValues { + badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + } + + tx, _ = dbwrap.NewTx(!writable, index, Txo{}) + defer tx.Rollback() + + n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64) + panicOn(err) + if n == 0 { + panic("containers should exist") + } + expected := uint64(len(putmeValues)) + if n != expected { + panic(fmt.Sprintf("expected CountRange() of %v but got n=%v", expected, n)) + } +} + +func TestBadger_CountRange_middle_container(t *testing.T) { + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_CountRange_middle_container") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + + putmeValues := []uint64{3, 2 << 16, 4 << 16} + + for _, putme := range putmeValues { + badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + } + + tx, _ := dbwrap.NewTx(!writable, index, Txo{}) + defer tx.Rollback() + + // pick out just the middle container with the 1 bit set on it. + n, err := tx.CountRange(index, field, view, shard, 4, (2<<16)+1) + panicOn(err) + if n != 1 { + panic("middle 1 bit container should exist") + } +} + +func TestBadger_CountRange_many_middle_container(t *testing.T) { + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_CountRange_many_middle_container") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + + putmeValues := []uint64{3, 2 << 16, 4 << 16} + + for _, putme := range putmeValues { + badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + } + + tx, _ := dbwrap.NewTx(!writable, index, Txo{}) + defer tx.Rollback() + + // get them all + n, err := tx.CountRange(index, field, view, shard, 0, (4<<16)+1) + panicOn(err) + if n != 3 { + panic("count should have been all 3 bits") + } +} + +func TestBadger_UnionInPlace(t *testing.T) { + + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_UnionInPlace") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + + putmeValues := []uint64{3, 2 << 16} + + others := roaring.NewBitmap() + others2 := roaring.NewBitmap() + others3 := roaring.NewBitmap() + // populate others with putmeValues +1 into others + + for _, putme := range putmeValues { + badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) + } + + tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) + n, err := tx2.Count(index, field, view, shard) + panicOn(err) + if n != 2 { + panic("should have 2 bits set") + } + tx2.Rollback() + + for _, putme := range putmeValues { + mustAddR(others.Add(putme)) // should not change count, b/c putme already in the rbm + mustAddR(others.Add(putme + 1)) + mustAddR(others2.Add(putme + 2)) + } + mustAddR(others3.Add(4 << 16)) // outside the 2<<16 container + + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + defer tx.Rollback() + err = tx.UnionInPlace(index, field, view, shard, others, others2, others3) + panicOn(err) + + // end game, check we got the union. + rbm, err := tx.RoaringBitmap(index, field, view, shard) + panicOn(err) + n = rbm.Count() + if n != 7 { + panic("should have a total 3 + 3 +1 = 7 bits set on the containers") + } +} + +func TestBadger_RoaringBitmap(t *testing.T) { + + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_RoaringBitmap") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + + expected := uint64(3) + putme := expected + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + + tx, _ := dbwrap.NewTx(!writable, index, Txo{}) + defer tx.Rollback() + + rbm, err := tx.RoaringBitmap(index, field, view, shard) + panicOn(err) + + slc := rbm.Slice() + if slc[0] != uint64(expected) { + panic(fmt.Sprintf("should have gotten %v back", expected)) + } +} + +func TestBadger_reverse_badger_iterator_and_prefix_valid(t *testing.T) { + + // does a reverse iterator and ValidForPrefix behave like we expect it too? + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_reverse_badger_iterator_and_prefix_valid") + defer clean() + defer dbwrap.Close() + + // add 0, 1, 2 to badgerdb as keys (and same value). + err := dbwrap.Update(func(txn *badger.Txn) error { + for _, prefix := range []string{"a", "b", "c"} { + for i := 0; i < 3; i++ { + kv := []byte(fmt.Sprintf("%v:%v", prefix, i)) + err := txn.Set(kv, kv) + panicOn(err) + } + } + return nil + }) + panicOn(err) + + tx0, _ := dbwrap.NewTx(!writable, "no-index-avail", Txo{}) + tx := tx0.(*BadgerTx) + + prefix := []byte("b:") + it := NewBadgerIterator(tx, prefix) + + if !it.it.Valid() { + panic("why is underlying badger it not valid here?") + } + results := "" + for it.Next() { + item := it.it.Item() + sk := string(item.Key()) + results += sk + ", " + } + expected := `b:0, b:1, b:2, ` + if results != expected { + panic(fmt.Sprintf("observed: '%v' but expected: '%v'", results, expected)) + } + it.Close() + + // now reversed + seekto := []byte("c:") + rit := NewBadgerReverseIterator(tx, prefix, seekto) // Seeks("b:") goes to b:0 + defer rit.Close() + + if !rit.it.Valid() { + panic("why is underlying badger it not valid here?") + } + results = "" + for rit.Next() { + item := rit.it.Item() + sk := string(item.Key()) + results += sk + ", " + } + expected = `b:2, b:1, b:0, ` + if results != expected { + panic(fmt.Sprintf("observed: '%v' but expected: '%v'", results, expected)) + } + rit.Close() +} + +func TestBadger_just_reverse_badger_iterator_and_prefix_valid(t *testing.T) { + + // does a reverse iterator and ValidForPrefix behave like we expect it too? + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_reverse_badger_iterator_and_prefix_valid") + defer clean() + defer dbwrap.Close() + + // add 0, 1, 2 to badgerdb as keys (and same value). + err := dbwrap.Update(func(txn *badger.Txn) error { + for _, prefix := range []string{"a", "b", "c"} { + for i := 0; i < 3; i++ { + kv := []byte(fmt.Sprintf("%v:%v", prefix, i)) + err := txn.Set(kv, kv) + panicOn(err) + } + } + return nil + }) + panicOn(err) + + tx0, _ := dbwrap.NewTx(!writable, "no-index-avail", Txo{}) + tx := tx0.(*BadgerTx) + + seekto := []byte("c:") + prefix := []byte("b:") + // now reversed + rit := NewBadgerReverseIterator(tx, prefix, seekto) // Seeks("b:") goes to b:0 + defer rit.Close() + + if !rit.it.Valid() { + panic("why is underlying badger rit not valid here?") + } + results := "" + for rit.Next() { + item := rit.it.Item() + sk := string(item.Key()) + results += sk + ", " + } + expected := `b:2, b:1, b:0, ` + if results != expected { + panic(fmt.Sprintf("observed: '%v' but expected: '%v'", results, expected)) + } + rit.Close() +} + +func TestBadger_ImportRoaringBits(t *testing.T) { + + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx0, _ := dbwrap.NewTx(writable, index, Txo{}) + tx := tx0.(*BadgerTx) + defer tx.Rollback() + tx.DeleteEmptyContainer = true // traditional badger Tx behavior, but not Roaring. + + //bitvalue := uint64(42) + + // get some roaring bits, get an itr RoaringIterator from them + rowSize := uint64(0) + //bits := []uint64{0} + bits := []uint64{0, 2, 5, 1<<16 + 1, 2 << 16} + data := getTestBitmapAsRawRoaring(bits...) + itr, err := roaring.NewRoaringIterator(data) + panicOn(err) + clear := false + logme := false + + changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) + _ = rowSet + if changed != len(bits) { + panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) + } + panicOn(err) + + for _, v := range bits { + exists, err := tx.Contains(index, field, view, shard, v) + panicOn(err) + if !exists { + panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) + } + } + + // now test the union in place with the same set gives no change. + + changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) + _ = rowSet + if changed != 0 { + panic(fmt.Sprintf("should have not changed any bits on the second import, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) + } + panicOn(err) + + for _, v := range bits { + exists, err := tx.Contains(index, field, view, shard, v) + panicOn(err) + if !exists { + panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) + } + } + + // now test the clear path + clear = true + + for _, v := range bits { + // clear 1 bit at a time + data := getTestBitmapAsRawRoaring(v) + itr, err := roaring.NewRoaringIterator(data) + panicOn(err) + + changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) + _ = rowSet + if changed != 1 { + panic(fmt.Sprintf("should have changed 1 bit: '%v', rowSet='%#v', err='%v'", changed, rowSet, err)) + } + panicOn(err) + } + n, err := tx.Count(index, field, view, shard) + panicOn(err) + if n != 0 { + panic(fmt.Sprintf("n = %v not zero so the clearbits didn't happen!", n)) + } + allkeys := stringifiedBadgerKeysTx(tx, false) + + // should have no keys + if allkeys != "" { + panic("badger should have no keys now") + } +} + +func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { + + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits_set_nonoverlapping_bits") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + defer tx.Rollback() + + // get some roaring bits, get an itr RoaringIterator from them + rowSize := uint64(0) + //bits := []uint64{0} + bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} + data := getTestBitmapAsRawRoaring(bits...) + itr, err := roaring.NewRoaringIterator(data) + panicOn(err) + + bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16} + data2 := getTestBitmapAsRawRoaring(bits2...) + itr2, err := roaring.NewRoaringIterator(data2) + panicOn(err) + + clear := false + logme := false + + changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) + _ = rowSet + if changed != len(bits) { + panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) + } + panicOn(err) + + for _, v := range bits { + exists, err := tx.Contains(index, field, view, shard, v) + panicOn(err) + if !exists { + panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) + } + } + + // now import the 2nd, overlapping set and set them. + + changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil) + _ = rowSet + if changed != 4 { + panic(fmt.Sprintf("should have changed 2 bits: the 1 and the 3, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) + } + panicOn(err) +} + +func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { + + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits_clear_nonoverlapping_bits") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + defer tx.Rollback() + + // get some roaring bits, get an itr RoaringIterator from them + rowSize := uint64(0) + //bits := []uint64{0} + bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} //, 5, 1<<16 + 1, 2 << 16} + data := getTestBitmapAsRawRoaring(bits...) + itr, err := roaring.NewRoaringIterator(data) + panicOn(err) + + bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16} + data2 := getTestBitmapAsRawRoaring(bits2...) + itr2, err := roaring.NewRoaringIterator(data2) + panicOn(err) + + clear := false + logme := false + + changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) + _ = rowSet + if changed != len(bits) { + panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) + } + panicOn(err) + + for _, v := range bits { + exists, err := tx.Contains(index, field, view, shard, v) + panicOn(err) + if !exists { + panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) + } + } + + // now import the 2nd overlapping set and clear them. + clear = true + + changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil) + _ = rowSet + if changed != 2 { + panic(fmt.Sprintf("should have changed 1 bit: the 2, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) + } + panicOn(err) + + n, err := tx.Count(index, field, view, shard) + panicOn(err) + if n != 2 { // just the 0 and the 1<<16 bits should be left set. + panic(fmt.Sprintf("n = %v not 2 so the clearbits didn't happen!", n)) + } + +} + +/* +func TestBadger_AutoCommit(t *testing.T) { + + // setup + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_AutoCommit") + defer clean() + defer dbwrap.Close() + + index, field, view, shard := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + + // if we go over 100K writes, we should autocommit + // rather than panic. + for v := 0; v < 133444; v++ { + changed, err := tx.Add(index, field, view, shard, doBatched, uint64(v)) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + } + + err := tx.Commit() + panicOn(err) +} + +func TestBadger_BigWritesAvoidTxnTooLargeWithAutoCommit(t *testing.T) { + + // setup + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_BigWritesAvoidTxnTooLargeWithAutoCommit") + defer clean() + defer dbwrap.Close() + + index, field, view, shard := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + + containerKey := uint64(0) + // setup + bits := make([]uint64, 1024) + n := 0 + for i := range bits { + bits[i] = ^uint64(0) + n += 64 + } + rc := roaring.NewContainerBitmap(n, bits) + + // if we go over 100K big writes, we should autocommit + // rather than panic. + for v := 0; v < 133444; v++ { + containerKey++ + err := tx.PutContainer(index, field, view, shard, containerKey, rc) + panicOn(err) + } + + err := tx.Commit() + panicOn(err) +} +*/ + +func TestBadger_DeleteIndex(t *testing.T) { + + // setup + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx0, _ := dbwrap.NewTx(writable, index, Txo{}) + tx := tx0.(*BadgerTx) + + bitvalue := uint64(777) + bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} + for _, v := range bits { + changed, err := tx.Add(index, field, view, shard, doBatched, v) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + } + + index2 := "i2" // should not be deleted, even though it shares a prefix with 'i' + changed, err := tx.Add(index2, field, view, shard, doBatched, bitvalue) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + + for _, v := range bits { + exists, err := tx.Contains(index, field, view, shard, v) + panicOn(err) + if !exists { + panic("ARG bitvalue was NOT SET!!!") + } + } + exists, err := tx.Contains(index2, field, view, shard, bitvalue) + panicOn(err) + if !exists { + panic("ARG bitvalue was NOT SET!!! on index2") + } + err = tx.Commit() + panicOn(err) + + // end of setup + err = dbwrap.DeleteIndex(index) + panicOn(err) + + tx0, _ = dbwrap.NewTx(!writable, index2, Txo{}) + tx = tx0.(*BadgerTx) + + defer tx.Rollback() + exists, err = tx.Contains(index2, field, view, shard, bitvalue) + panicOn(err) + if !exists { + panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2)) + } + + for _, v := range bits { + exists, err = tx.Contains(index, field, view, shard, v) + panicOn(err) + if exists { + allkeys := stringifiedBadgerKeysTx(tx, false) + panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) + } + } +} + +func TestBadger_DeleteIndex_over100k(t *testing.T) { + t.Skip("test big and long running, skip") + // setup + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex_over100k") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx0, _ := dbwrap.NewTx(writable, index, Txo{}) + tx := tx0.(*BadgerTx) + + bitvalue := uint64(777) + limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction. + //limit := uint64(101) + for v := uint64(1); v < limit; v++ { + // shift by << 16 to get into a different shard + changed, err := tx.Add(index, field, view, shard, doBatched, v<<16) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + if v%100000 == 0 { + panicOn(tx.Commit()) + tx0, _ := dbwrap.NewTx(writable, index, Txo{}) + tx = tx0.(*BadgerTx) + } + } + + index2 := "i2" // should not be deleted, even though it shares a prefix with 'i' + changed, err := tx.Add(index2, field, view, shard, doBatched, bitvalue) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + err = tx.Commit() + panicOn(err) + + // end of setup + err = dbwrap.DeleteIndex(index) + panicOn(err) + + tx0, _ = dbwrap.NewTx(!writable, index2, Txo{}) + tx = tx0.(*BadgerTx) + + defer tx.Rollback() + exists, err := tx.Contains(index2, field, view, shard, bitvalue) + panicOn(err) + if !exists { + panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2)) + } + + for v := uint64(0); v < limit; v++ { + exists, err = tx.Contains(index, field, view, shard, v<<16) + panicOn(err) + if exists { + allkeys := stringifiedBadgerKeysTx(tx, false) + panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) + } + } +} + +func TestBitmapDiff(t *testing.T) { + a := roaring.NewBitmap() + b := roaring.NewBitmap() + err := roaringBitmapDiff(a, b) + panicOn(err) + err = roaringBitmapDiff(b, a) + panicOn(err) + + a = roaring.NewBitmap(0) + err = roaringBitmapDiff(a, b) + if err == nil { + panic("diff should have been noticed") + } + err = roaringBitmapDiff(b, a) + if err == nil { + panic("diff should have been noticed") + } + + b = roaring.NewBitmap(0) + err = roaringBitmapDiff(a, b) + panicOn(err) + err = roaringBitmapDiff(b, a) + panicOn(err) + + a = roaring.NewBitmap() + + err = roaringBitmapDiff(a, b) + if err == nil { + panic("diff should have been noticed") + } + err = roaringBitmapDiff(b, a) + if err == nil { + panic("diff should have been noticed") + } + + a = roaring.NewBitmap(1) + + err = roaringBitmapDiff(a, b) + if err == nil { + panic("diff should have been noticed") + } + err = roaringBitmapDiff(b, a) + if err == nil { + panic("diff should have been noticed") + } + + b = roaring.NewBitmap(1, 2) + a = roaring.NewBitmap(0, 1) + + err = roaringBitmapDiff(a, b) + if err == nil { + panic("diff should have been noticed") + } + err = roaringBitmapDiff(b, a) + if err == nil { + panic("diff should have been noticed") + } + + b = roaring.NewBitmap(1, 2, 3) + a = roaring.NewBitmap(1, 2) + + err = roaringBitmapDiff(a, b) + if err == nil { + panic("diff should have been noticed") + } + err = roaringBitmapDiff(b, a) + if err == nil { + panic("diff should have been noticed") + } +} + +func TestBadger_SliceOfShards(t *testing.T) { + + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SliceOfShards") + defer clean() + defer dbwrap.Close() + index, field, view := "i", "f", "v" + shards := []uint64{0, 1, 2, 3, 1000001, 2000001} + putme := uint64(179) + for _, shard := range shards { + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + } + tx, _ := dbwrap.NewTx(!writable, index, Txo{}) + defer tx.Rollback() + + slc, err := tx.SliceOfShards(index, field, view, "") + panicOn(err) + for i := range shards { + if shards[i] != slc[i] { + panic(fmt.Sprintf("expected at i=%v that slc[i]=%v = shards[i]=%v", i, slc[i], shards[i])) + } + } +} + +func TestBadger_HasData(t *testing.T) { + + db, clean := mustOpenEmptyBadgerWrapper("TestBadger_HasData") + defer clean() + defer db.Close() + + // HasData should start out false. + hasAnything, err := db.HasData() + if err != nil { + t.Fatal(err) + } + if hasAnything { + t.Fatalf("HasData reported existing data on an empty database") + } + + // check that HasData sees a committed record. + + index, field, view, shard, putme := "i", "f", "v", uint64(123), uint64(42) + badgerDBMustSetBitvalue(db, index, field, view, shard, putme) + + // HasData(false) should now report data + hasAnything, err = db.HasData() + if err != nil { + t.Fatal(err) + } + if !hasAnything { + t.Fatalf("HasData() reported no data on a database that has bits written to it") + } +} + +// Benchmark performance of setValue for BSI ranges. +func BenchmarkBadger_Write(b *testing.B) { + + dbwrap, clean := mustOpenEmptyBadgerWrapper("BenchmarkBadger_Write") + //defer clean() + _ = clean + defer dbwrap.Close() + + putmeValues := []uint64{3, 2 << 16} + index, field, view, shard := "i", "f", "v", uint64(0) + + for _, putme := range putmeValues { + badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) + } + /* + + dbwrap, clean := mustOpenEmptyBadgerWrapper("BenchmarkBadger_Write") + defer clean() + defer dbwrap.Close() + index, field, view, shard := "i", "f", "v", uint64(0) + tx, _ := dbwrap.NewTx(writable, index, Txo{}) + + bitvalue := uint64(1 << 20) + changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + + bitvalue2 := uint64(1<<20 + 1) + changed, err = tx.Add(index, field, view, shard, doBatched, bitvalue2) + if changed <= 0 { + panic("should have changed") + } + panicOn(err) + + exists, err := tx.Contains(index, field, view, shard, bitvalue) + panicOn(err) + if !exists { + panic("ARG bitvalue was NOT SET!!!") + } + exists, err = tx.Contains(index, field, view, shard, bitvalue2) + panicOn(err) + if !exists { + panic("ARG bitvalue2 was NOT SET!!!") + } + + err = tx.Commit() + panicOn(err) + + offset := uint64(0 << 20) + start := uint64(0 << 16) + endx := bitvalue + 1<<16 + + tx2 := dbwrap.NewTx(!writable, index, Txo{}) + rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx) + panicOn(err) + tx2.Rollback() + + // should see our 1M value + s2 := bitmapAsString(rbm2) + expect2 := "c(1048576, 1048577)" + if s2 != expect2 { + panic(fmt.Sprintf("s2='%v', but expected '%v'", s2, expect2)) + } + + // now offset by 2M + offset = uint64(2 << 20) + tx3 := dbwrap.NewTx(!writable, index, Txo{}) + rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx) + panicOn(err) + tx3.Rollback() + + //expect to see 3M == 3145728 + s3 := bitmapAsString(rbm3) + expect3 := "c(3145728, 3145729)" + + if s3 != expect3 { + panic(fmt.Sprintf("s3='%v', but expected '%v'", s3, expect3)) + } + */ +} + +func reportTestBadgersNeedingClose() error { + globalBadgerReg.mu.Lock() + defer globalBadgerReg.mu.Unlock() + n := len(globalBadgerReg.mp) + if n == 0 { + return nil + } + AlwaysPrintf("*** these badgers are still open (n=%v):", n) + i := 0 + for w := range globalBadgerReg.mp { + AlwaysPrintf("i=%v, w p=%p stack:\n%v\n\n", i, w, w.startStack) + i++ + } + return errors.New("unclosed badgers, contact Animal Control") +} + +/* +func TestBadger_ConflictWriteWriteResolution(t *testing.T) { + + // 1) when do we get write-write conflicts (probably different goroutines) but + // can we get them on different keys? + + // 2) does having a lock registry that insures we are only ever writing + // different keys at once avoid write-write conflicts? + + // 3) how should write-write conflicts be resolved? + // presumably just retying the write? + + // setup + dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ConflictWriteWriteResolution") + defer clean() + defer dbwrap.Close() + + concur := 10 + //bkey := []byte("a") + //by := []byte("value-for-a") + + // read-loop: + for i := 0; i < concur*2; i++ { + go func() { + tx, _ := dbwrap.NewTx(!writable, "", Txo{}) + + for j := 0; true; j++ { + + bkey := []byte(fmt.Sprintf("key-for-a j=%v", j)) + //by := []byte(fmt.Sprintf("value-for-a j=%v", -1)) + + _, err := tx.tx.Get(bkey) + if err != badger.ErrKeyNotFound { + panicOn(err) + } + + if j%10 == 0 { + vv("committing after 10, gid=%v", curGID()) + tx.Rollback() + tx, _ = dbwrap.NewTx(!writable, "", Txo{}) + } + + } + }() + } + + // write-loop: + for i := 0; i < concur; i++ { + go func() { + tx, _ := dbwrap.NewTx(writable, "", Txo{}) + + for j := 0; true; j++ { + + bkey := []byte(fmt.Sprintf("key-for-a j=%v", j)) + by := []byte(fmt.Sprintf("value-for-a j=%v", j)) + + entry := badger.NewEntry(bkey, by) + err := tx.tx.SetEntry(entry) + panicOn(err) + + if j%5 == 0 { + panicOn(tx.tx.Delete(bkey)) + } + + _, err = tx.tx.Get(bkey) + if err != badger.ErrKeyNotFound { + panicOn(err) + } + + if j%10 == 0 { + ///vv("committing after 10, gid=%v", curGID()) + err = tx.tx.Commit() + panicOn(err) + tx, _ = dbwrap.NewTx(writable, "", Txo{}) + } + + } + }() + } + select {} +} +*/ diff --git a/bluegreentx.go b/bluegreentx.go index 1b7ad72c7..033f2809c 100644 --- a/bluegreentx.go +++ b/bluegreentx.go @@ -32,7 +32,8 @@ import ( // The checkDatabase() call will do reads of the fragments at Commit/Rollback, // while the snapshotqueue may be doing writes. // -// Do not run with go test -race and expect it to be race free. +// Do not run with go test -race and expect it to be race free with RoaringTx +// on one arm. // type blueGreenTx struct { a Tx @@ -42,7 +43,8 @@ type blueGreenTx struct { as string bs string - types []txtype + types []txtype + hasRoaring bool // roaring will not create as many Tx (they are // psuedo Tx anyway), espcially when deleting @@ -57,6 +59,8 @@ type blueGreenTx struct { rollbackOrCommitDone bool txf *TxFactory + + short bool // short Dump or long } // blueGreenRegistry is used to force checking of (read) transactions @@ -65,19 +69,29 @@ type blueGreenTx struct { // roaring will show up, while writes to the DB won't show up on // readTx that have already started. type blueGreenRegistry struct { - mu sync.Mutex - m map[int64]*blueGreenTx + mu sync.Mutex + m map[int64]*blueGreenTx + types []txtype + hasRoaring bool } -func newBlueGreenReg() *blueGreenRegistry { +// if we have raoring in the mix we cannot expect reads +// to match up, but otherwise do. +func newBlueGreenReg(types []txtype) *blueGreenRegistry { + + hasRoaring := false + if types[0] == roaringTxn || types[1] == roaringTxn { + hasRoaring = true + } return &blueGreenRegistry{ - m: make(map[int64]*blueGreenTx), + m: make(map[int64]*blueGreenTx), + types: types, + hasRoaring: hasRoaring, } } -// add remembers the tx so we can check it -// should we see a write after creation -// but before rollback/commit. +// add remembers the tx so we can check that +// all tx were finished before Close(). func (b *blueGreenRegistry) add(c *blueGreenTx) { b.mu.Lock() defer b.mu.Unlock() @@ -100,26 +114,33 @@ func (b *blueGreenRegistry) Close() { b.mu.Lock() defer b.mu.Unlock() if len(b.m) > 0 { - panic(fmt.Sprintf("still have unchecked blueGreenTx: '%#v'", b.m)) + panic(fmt.Sprintf("still have open/unchecked blueGreenTx: '%#v'", b.m)) //AlwaysPrintf("still have unchecked blueGreenTx: '%#v'", b.m) } } func (txf *TxFactory) newBlueGreenTx(a, b Tx, idx *Index, o Txo) *blueGreenTx { - //func (reg *blueGreenRegistry) newBlueGreenTx(a, b Tx, idx *Index, o Txo, openTxSn []int64) *blueGreenTx { as := a.Type() bs := b.Type() - c := &blueGreenTx{a: a, b: b, idx: idx, as: as, bs: bs, txf: txf, types: txf.types} + c := &blueGreenTx{a: a, + b: b, + idx: idx, + as: as, + bs: bs, + txf: txf, + types: txf.types, + hasRoaring: txf.blueGreenReg.hasRoaring, + short: true, + } if c.types[1] == roaringTxn { c.useSnA = true } + //vv("newBlueGreenTx with a.sn=%v with o.Shard=%v", c.Sn(), int(o.Shard)) c.checker.c = c c.o = o - if o.Write { - txf.blueGreenReg.add(c) - } + txf.blueGreenReg.add(c) return c } @@ -131,24 +152,27 @@ func (c *blueGreenTx) Type() string { var blueGreenTxDumpMut sync.Mutex -func (c *blueGreenTx) Dump() { +func (c *blueGreenTx) Dump(short bool) { + blueGreenTxDumpMut.Lock() defer blueGreenTxDumpMut.Unlock() fmt.Printf("%v blueGreenTx.Dump ============== \n", FileLine(2)) fmt.Printf("A(%v) Dump:\n", c.as) - c.a.Dump() + c.a.Dump(short) fmt.Printf("B(%v) Dump:\n", c.bs) - c.b.Dump() + c.b.Dump(short) - fmt.Printf("dbPerShard.DumpAll(): idx=%p\n", c.idx) - c.idx.Txf.dbPerShard.DumpAll() + if !short { + fmt.Printf("dbPerShard.DumpAll(): idx=%p\n", c.idx) + c.idx.holder.txf.dbPerShard.DumpAll() + } } func (c *blueGreenTx) Readonly() bool { a := c.a.Readonly() b := c.b.Readonly() if a != b { - panic(fmt.Sprintf("a=%v, but b =%v", a, b)) + panic(fmt.Sprintf("Readonly difference, a=%v, but b =%v", a, b)) } return b } @@ -172,6 +196,9 @@ func (c *blueGreenTx) IncrementOpN(index, field, view string, shard uint64, chan // compareTxState is called for the first Commit or Rollback a blueGreenTx sees. func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) { + if c.o.blueGreenOff { + return + } here := fmt.Sprintf("%v/%v/%v/%v", index, field, view, shard) //vv("compareTxState here = '%v', _sn_ %v gid=%v", here, c.Sn(), curGID()) aIter, aFound, aErr := c.a.ContainerIterator(index, field, view, shard, 0) @@ -184,21 +211,21 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) { } if aFound != bFound { - c.Dump() + c.Dump(c.short) panic(fmt.Sprintf("compareTxState[%v]: A(%v) ContainerIterator had aFound=%v, but B(%v) had bFound=%v; at '%v'", here, c.as, aFound, c.bs, bFound, stack())) } if aErr != nil || bErr != nil { if aErr != nil && bErr != nil { - c.Dump() + c.Dump(c.short) panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err '%v'; B(%v) reported err '%v' at %v", here, c.as, aErr, c.bs, bErr, stack())) } if aErr != nil { - c.Dump() + c.Dump(c.short) panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err %v at %v; but B(%v) did not", here, c.as, aErr, c.bs, stack())) } if bErr != nil { - c.Dump() + c.Dump(c.short) panic(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, stack())) } } @@ -207,17 +234,17 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) { if !bIter.Next() { AlwaysPrintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, dump to follow, stack=\n %v\n\n and here is dump:", here, c.as, aKey, c.bs, stack()) - c.Dump() + c.Dump(c.short) panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, stack())) } bKey, bValue := bIter.Value() if bKey != aKey { AlwaysPrintf("problem in caller %v", Caller(2)) - c.Dump() + c.Dump(c.short) panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, stack())) } if err := aValue.BitwiseCompare(bValue); err != nil { - c.Dump() + c.Dump(c.short) //vv("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack()) panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack())) } @@ -226,7 +253,7 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) { // end checking everything in A, but does B have more? if bIter.Next() { AlwaysPrintf("bIter has more than it should. problem in caller %v. _sn_ %v", Caller(2), c.Sn()) - c.Dump() + c.Dump(c.short) bKey, _ := bIter.Value() panic(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, (a.sn=%v) (b.sn=%v) at %v", here, c.bs, bKey, c.as, c.a.Sn(), c.b.Sn(), stack())) } @@ -234,10 +261,17 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) { } func (c *blueGreenTx) checkDatabase() { - if !c.o.Write { - // We only need to check the we are A/B consistent after every write. - // Then reads can only see that consitent state, and don't need - // to be checked themselves. Sketch of proof by induction: + if c.o.blueGreenOff { + return + } + if c.hasRoaring && !c.o.Write { + // With roaring on one arm, we only check the we are A/B + // consistent after every write. + // + // Ideally reads can only see that consitent state, and don't need + // to be checked themselves-- but we do try if both A and B + // are transactional. Sketch of proof by induction that + // write checking should, theoretically, suffice: // Starting with zero data, if we have agreement in both A/B // database state after each write, then // because there is only ever a single @@ -245,7 +279,8 @@ func (c *blueGreenTx) checkDatabase() { // data state between A and B as long as every prior // A/B check of the serialized writes suceeded. // - // This avoids a key problem we discovered when A/B checking reads. + // This avoids a key problem we discovered when A/B checking reads + // with roaring on one arm. // The MVCC of the transactional engines means that reads that // start before a write commit will look very different // when comparing to roaring's non-transactional state. @@ -259,8 +294,6 @@ func (c *blueGreenTx) checkDatabase() { } c.checker.checkDone = true - // seen() returns nil on 2nd or any further call, - // so only the first Commit() or Rollback() does this. for index, fields := range c.checker.seen() { for field, views := range fields { for view, shards := range views { @@ -272,6 +305,10 @@ func (c *blueGreenTx) checkDatabase() { } } +func (c *blueGreenTx) IsDone() bool { + return c.b.IsDone() +} + func (c *blueGreenTx) Rollback() { c.mu.Lock() defer c.mu.Unlock() @@ -308,7 +345,9 @@ func (c *blueGreenTx) Commit() error { //vv("blueGreenTx.Commit() called. bgtx p=%p", c) c.rollbackOrCommitDone = true if c.o.Write { - c.checkDatabase() + if !c.o.blueGreenOff { + c.checkDatabase() + } } defer func() { if r := recover(); r != nil { @@ -336,14 +375,15 @@ func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*r a, errA := c.a.RoaringBitmap(index, field, view, shard) _, _ = a, errA b, errB := c.b.RoaringBitmap(index, field, view, shard) - compareErrors(errA, errB) + if !c.o.blueGreenOff { + compareErrors(errA, errB) - slcA := a.Slice() - slcB := b.Slice() - if !reflect.DeepEqual(slcA, slcB) { - panic("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!") + slcA := a.Slice() + slcB := b.Slice() + if !reflect.DeepEqual(slcA, slcB) { + panic("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!") + } } - return b, errB } @@ -357,10 +397,12 @@ func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uin }() a, errA := c.a.Container(index, field, view, shard, key) b, errB := c.b.Container(index, field, view, shard, key) - compareErrors(errA, errB) - err = a.BitwiseCompare(b) - panicOn(err) + if !c.o.blueGreenOff { + compareErrors(errA, errB) + err = a.BitwiseCompare(b) + panicOn(err) + } return b, errB } @@ -374,8 +416,10 @@ func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key }() errA := c.a.PutContainer(index, field, view, shard, key, rc) errB := c.b.PutContainer(index, field, view, shard, key, rc) - compareErrors(errA, errB) + if !c.o.blueGreenOff { + compareErrors(errA, errB) + } return errB } @@ -386,7 +430,7 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, // ================== begin save comments. //c.checkDatabase() ////vv("got past database check at TOP of ImportRoaringBits") - //c.Dump() + //c.Dump(c.short) ////vv("done with top dump; clear=%v", clear) // ================== end save comments. defer func() { @@ -403,28 +447,31 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize, data) - if len(data) == 0 { - // okay to check! otherwise we are in the fragment.fillFragmentFromArchive - // case where we know that RoaringTx.ImportRoaringBits changed and rowSet will - // be inaccurate. - if changedA != changedB { - panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB)) - } - if len(rowSetA) != len(rowSetB) { - panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB)) - } - for k, va := range rowSetA { - vb, ok := rowSetB[k] - if !ok { - panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB)) + if !c.o.blueGreenOff { + + if len(data) == 0 { + // okay to check! otherwise we are in the fragment.fillFragmentFromArchive + // case where we know that RoaringTx.ImportRoaringBits changed and rowSet will + // be inaccurate. + if changedA != changedB { + panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB)) } - if va != vb { - panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb)) + if len(rowSetA) != len(rowSetB) { + panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB)) + } + for k, va := range rowSetA { + vb, ok := rowSetB[k] + if !ok { + panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB)) + } + if va != vb { + panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb)) + } } } + compareErrors(errA, errB) + c.checkDatabase() } - compareErrors(errA, errB) - c.checkDatabase() return changedB, rowSetB, errB } @@ -438,7 +485,10 @@ func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, k }() errA := c.a.RemoveContainer(index, field, view, shard, key) errB := c.b.RemoveContainer(index, field, view, shard, key) - compareErrors(errA, errB) + + if !c.o.blueGreenOff { + compareErrors(errA, errB) + } return errB } @@ -481,10 +531,14 @@ func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool, bch, errB := c.b.Add(index, field, view, shard, batched, a2...) - if ach != bch { - panic(fmt.Sprintf("Add() difference, ach=%v, but bch=%v; errA='%v'; errB='%v'", ach, bch, errA, errB)) + if !c.o.blueGreenOff { + + if ach != bch { + panic(fmt.Sprintf("Add() difference, ach=%v, but bch=%v; errA='%v'; errB='%v'", ach, bch, errA, errB)) + } + compareErrors(errA, errB) } - compareErrors(errA, errB) + return bch, errB } @@ -516,7 +570,10 @@ func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint6 ach, errA := c.a.Remove(index, field, view, shard, a...) _, _ = ach, errA bch, errB := c.b.Remove(index, field, view, shard, a...) - compareErrors(errA, errB) + + if !c.o.blueGreenOff { + compareErrors(errA, errB) + } return bch, errB } @@ -532,7 +589,9 @@ func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint _, _ = ax, errA bx, errB := c.b.Contains(index, field, view, shard, key) - compareErrors(errA, errB) + if !c.o.blueGreenOff { + compareErrors(errA, errB) + } return bx, errB } @@ -550,8 +609,10 @@ func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64, bit, bfound, errB := c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) - compareErrors(errA, errB) - // INVAR: errA == errB, so only need to check one. + if !c.o.blueGreenOff { + compareErrors(errA, errB) + } + if errB != nil { // RoaringTx can return an iterator and an error, so be sure Close it we have it. if ait != nil { @@ -562,6 +623,13 @@ func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64, } return nil, bfound, errB } + if errA != nil { + // RoaringTx can return an iterator and an error, so be sure Close it we have it. + if ait != nil { + ait.Close() + } + } + // INVAR: errA == errB == nil bgi := NewBlueGreenIterator(c, ait, bit) return bgi, bfound, errB @@ -598,11 +666,14 @@ func (bgi *blueGreenIterator) Next() bool { func (bgi *blueGreenIterator) Value() (uint64, *roaring.Container) { ka, ca := bgi.ait.Value() kb, cb := bgi.bit.Value() - if ka != kb { - panic(fmt.Sprintf("ka=%v != kb=%v", ka, kb)) + + if !bgi.tx.o.blueGreenOff { + if ka != kb { + panic(fmt.Sprintf("ka=%v != kb=%v", ka, kb)) + } + err := ca.BitwiseCompare(cb) + panicOn(err) } - err := ca.BitwiseCompare(cb) - panicOn(err) return kb, cb } func (bgi *blueGreenIterator) Close() { @@ -611,7 +682,7 @@ func (bgi *blueGreenIterator) Close() { } // ForEach is read-only on the database, and so we only pass through to B. -// Avoids the side-effects of calling fn too many times. +// Avoids the side-effects of calling fn too many times, which can cause serious false alarms. func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { defer func() { if r := recover(); r != nil { @@ -654,7 +725,9 @@ func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, er b, errB := c.b.Count(index, field, view, shard) _, _ = b, errB - compareErrors(errA, errB) + if !c.o.blueGreenOff { + compareErrors(errA, errB) + } return b, errB } @@ -671,7 +744,9 @@ func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, erro b, errB := c.b.Max(index, field, view, shard) _, _ = b, errB - compareErrors(errA, errB) + if !c.o.blueGreenOff { + compareErrors(errA, errB) + } return b, errB } @@ -688,7 +763,9 @@ func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool bmin, bfound, errB := c.b.Min(index, field, view, shard) _, _, _ = bmin, bfound, errB - compareErrors(errA, errB) + if !c.o.blueGreenOff { + compareErrors(errA, errB) + } return bmin, bfound, errB } @@ -702,7 +779,9 @@ func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, othe }() errA := c.a.UnionInPlace(index, field, view, shard, others...) errB := c.b.UnionInPlace(index, field, view, shard, others...) - compareErrors(errA, errB) + if !c.o.blueGreenOff { + compareErrors(errA, errB) + } return errB } @@ -710,7 +789,7 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - c.Dump() + c.Dump(c.short) AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack()) panic(r) } @@ -718,11 +797,13 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, a, errA := c.a.CountRange(index, field, view, shard, start, end) b, errB := c.b.CountRange(index, field, view, shard, start, end) - if a != b { - panic(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b)) - } + if !c.o.blueGreenOff { + if a != b { + panic(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b)) + } - compareErrors(errA, errB) + compareErrors(errA, errB) + } return b, errB } @@ -730,19 +811,22 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack()) + AlwaysPrintf("see OffsetRange() on _sn_ %v, panic '%v' at '%v'", c.Sn(), r, stack()) panic(r) } }() a, errA := c.a.OffsetRange(index, field, view, shard, offset, start, end) b, errB := c.b.OffsetRange(index, field, view, shard, offset, start, end) - err = roaringBitmapDiff(a, b) - if err != nil { - c.Dump() - panicOn(err) + if !c.o.blueGreenOff { + + err = roaringBitmapDiff(a, b) + if err != nil { + c.Dump(false) + panicOn(fmt.Errorf("on _sn_ %v OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) err: %v", c.Sn(), index, field, view, int(shard), offset, start, end, err)) + } + compareErrors(errA, errB) } - compareErrors(errA, errB) return b, errB } @@ -751,7 +835,7 @@ func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint6 c.checker.see(index, field, view, shard) defer func() { if r := recover(); r != nil { - c.Dump() + c.Dump(c.short) AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack()) panic(r) } @@ -760,14 +844,19 @@ func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint6 rcA, szA, errA := c.a.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) rcB, szB, errB := c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) - compareErrors(errA, errB) + if !c.o.blueGreenOff { + compareErrors(errA, errB) + } // We are seeing Roaring vs Badger size differences on // server/ test TestClusterResize_AddNode/ContinuousShards, // so turn off the szA vs szB checks and MutliReaderB use. But keep them if we want to // check RBF vs Badger for byte-for-byte compatiblity (we // suspect the ops log or optimized bitmaps are accounting for the difference). - sizeMustMatch := false + sizeMustMatch := false // !c.hasRoaring + if c.o.blueGreenOff { + sizeMustMatch = false + } if sizeMustMatch { if szA != szB { panic(fmt.Sprintf("szA(%v) = %v, but szB(%v) = %v; fragmentPathForRoaring='%v'", c.as, szA, c.bs, szB, fragmentPathForRoaring)) @@ -806,41 +895,44 @@ func (c *blueGreenTx) SliceOfShards(index, field, view, optionalViewPath string) //c.checker.see(index, field, view, shard) // don't have shard. defer func() { if r := recover(); r != nil { - c.Dump() + c.Dump(c.short) AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack()) panic(r) } }() slcA, errA := c.a.SliceOfShards(index, field, view, optionalViewPath) slcB, errB := c.b.SliceOfShards(index, field, view, optionalViewPath) - compareErrors(errA, errB) - // sort order may be different, and that's ok. - cpa := append([]uint64{}, slcA...) - cpb := append([]uint64{}, slcB...) - sort.Slice(cpa, func(i, j int) bool { return cpa[i] < cpa[j] }) - sort.Slice(cpb, func(i, j int) bool { return cpb[i] < cpb[j] }) + if !c.o.blueGreenOff { + compareErrors(errA, errB) - if !reflect.DeepEqual(cpa, cpb) { - // report the first difference - ma := make(map[uint64]bool) - for _, ka := range slcA { - ma[ka] = true - } - for _, kb := range slcB { - if !ma[kb] { - //vv("blueGreenTx SliceOfShards diference! B(%v) had shard %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb) - c.Dump() - panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B(%v) had shard %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb)) + // sort order may be different, and that's ok. + cpa := append([]uint64{}, slcA...) + cpb := append([]uint64{}, slcB...) + sort.Slice(cpa, func(i, j int) bool { return cpa[i] < cpa[j] }) + sort.Slice(cpb, func(i, j int) bool { return cpb[i] < cpb[j] }) + + if !reflect.DeepEqual(cpa, cpb) { + // report the first difference + ma := make(map[uint64]bool) + for _, ka := range slcA { + ma[ka] = true } - delete(ma, kb) - } - if len(ma) != 0 { - for firstDifference := range ma { - panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! A(%v) had %v, but B(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.as, firstDifference, c.bs, cpa, cpb)) + for _, kb := range slcB { + if !ma[kb] { + //vv("blueGreenTx SliceOfShards diference! B(%v) had shard %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb) + c.Dump(c.short) + panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B(%v) had shard %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb)) + } + delete(ma, kb) } + if len(ma) != 0 { + for firstDifference := range ma { + panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! A(%v) had %v, but B(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.as, firstDifference, c.bs, cpa, cpb)) + } + } + panic(fmt.Sprintf("blueGreenTx SliceOfShards diference \n slcA(%v)='%#v';\n slcB(%v)='%#v';\n", c.as, cpa, c.bs, cpb)) } - panic(fmt.Sprintf("blueGreenTx SliceOfShards diference \n slcA(%v)='%#v';\n slcB(%v)='%#v';\n", c.as, cpa, c.bs, cpb)) } return slcB, errB } diff --git a/bluegreentx_test.go b/bluegreentx_test.go index 131712450..7455f2c6d 100644 --- a/bluegreentx_test.go +++ b/bluegreentx_test.go @@ -16,13 +16,20 @@ package pilosa import ( "bytes" + "context" "io" "io/ioutil" + "os" + "strings" "testing" cryrand "crypto/rand" ) +var _ = context.Background +var _ = os.Open +var _ = strings.Split + func TestMultiReaderB(t *testing.T) { // MultiReaderB should read identical chunks of bytes from both its "a" and "b" // member io.Readers, else it should panic. This should hold for diff --git a/catcher.go b/catcher.go index 8ff120f57..e2b4766e5 100644 --- a/catcher.go +++ b/catcher.go @@ -58,8 +58,8 @@ func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, r return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) } -func (c *catcherTx) Dump() { - c.b.Dump() +func (c *catcherTx) Dump(short bool) { + c.b.Dump(short) } func (c *catcherTx) Readonly() bool { @@ -145,6 +145,10 @@ func (c *catcherTx) UseRowCache() bool { return c.b.UseRowCache() } +func (c *catcherTx) IsDone() bool { + return c.b.IsDone() +} + func (c *catcherTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) { defer func() { diff --git a/cluster.go b/cluster.go index 56cb417ab..61a2babc7 100644 --- a/cluster.go +++ b/cluster.go @@ -62,7 +62,7 @@ const ( resizeJobActionAdd = "ADD" resizeJobActionRemove = "REMOVE" - defaultConfirmDownRetries = 10 + defaultConfirmDownRetries = 120 defaultConfirmDownSleep = 1 * time.Second ) @@ -758,7 +758,7 @@ func (c *cluster) fragsByHost(idx *Index) fragsByHost { fieldViews.addView(field.Name(), view.name) } } - return c.fragCombos(idx.Name(), idx.AvailableShards(), fieldViews) + return c.fragCombos(idx.Name(), idx.AvailableShards(includeRemote), fieldViews) } // fragCombos returns a map (by uri) of lists of fragments for a given index @@ -2218,7 +2218,7 @@ func (c *cluster) nodeStatus() *NodeStatus { is := &IndexStatus{Name: idx.Name, CreatedAt: idx.CreatedAt} for _, f := range idx.Fields { if field := c.holder.Field(idx.Name, f.Name); field != nil { - availableShards = field.AvailableShards() + availableShards = field.AvailableShards(includeRemote) } else { availableShards = roaring.NewBitmap() } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index e7c246fc7..003e7f668 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -97,6 +97,7 @@ func newIndexWithTempPath(tb testing.TB, name string) *Index { panic(err) } h := NewHolder(path, nil) + panicOn(h.Open()) index, err := h.CreateIndex(name, IndexOptions{}) testhook.Cleanup(tb, func() { h.Close() @@ -170,7 +171,7 @@ func TestFragSources(t *testing.T) { // Obtain transaction. var shard uint64 - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) defer tx.Rollback() _, err = field.SetBit(tx, 1, 101, nil) @@ -180,7 +181,7 @@ func TestFragSources(t *testing.T) { panicOn(tx.Commit()) shard = 1 - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) defer tx.Rollback() _, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil) if err != nil { @@ -189,7 +190,7 @@ func TestFragSources(t *testing.T) { panicOn(tx.Commit()) shard = 2 - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) defer tx.Rollback() _, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil) @@ -199,7 +200,7 @@ func TestFragSources(t *testing.T) { panicOn(tx.Commit()) shard = 3 - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) defer tx.Rollback() _, err = field.SetBit(tx, 1, ShardWidth*shard+1, nil) diff --git a/dbshard.go b/dbshard.go index 50d74e372..f7f629ba1 100644 --- a/dbshard.go +++ b/dbshard.go @@ -22,6 +22,9 @@ import ( "strconv" "strings" "sync" + "sync/atomic" + + "github.com/pkg/errors" ) var _ = sort.Sort @@ -50,6 +53,9 @@ type DBWrapper interface { DeleteField(index, field, fieldPath string) error OpenListString() string OpenSnList() (sns []int64) + Path() string + HasData() (has bool, err error) + SetHolder(h *Holder) } type DBRegistry interface { @@ -68,7 +74,9 @@ type DBShard struct { // by a reader to start with. mut sync.RWMutex - types []txtype + types []txtype + hasRoaring bool // if either of the types is roaringTxn + W []DBWrapper ParentDBIndex *DBIndex @@ -76,6 +84,7 @@ type DBShard struct { per *DBPerShard useOpenList int + closed bool } func (dbs *DBShard) DeleteFragment(index, field, view string, shard uint64, frag interface{}) (err error) { @@ -105,6 +114,7 @@ func (dbs *DBShard) Close() (err error) { return err } } + dbs.closed = true return } @@ -112,10 +122,49 @@ func (dbs *DBShard) String() string { return dbs.Path } -func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { - dbs.mut.Lock() - defer dbs.mut.Unlock() +// Cleanup must be called at every commit/rollback of a Tx, in +// order to release the read-write mutex that guarantees a single +// writer at a time. Each tx must take care to call cleanup() +// exactly once. examples: +// tx.o.dbs.Cleanup(tx) +// tx.Options().dbs.Cleanup(tx) +// +func (dbs *DBShard) Cleanup(tx Tx) { + if dbs == nil { + return // some tests are using Tx only, no dbs available. + } + if useRWLock { + if !dbs.hasRoaring { + if tx.Readonly() { + dbs.mut.RUnlock() + } else { + dbs.mut.Unlock() + } + } + } +} +// experimental feature, off for now. +const useRWLock = false + +func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { + if useRWLock { + // enforce only one writer at a time. The dbs.mut is held until + // the Tx finishes. + if !dbs.hasRoaring { + if write { + dbs.mut.Lock() + } else { + dbs.mut.RLock() + } + } + } + if o.dbs != dbs { + panic(fmt.Sprintf("TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs)) + } + if o.Shard != dbs.Shard { + panic(fmt.Sprintf("shard disagreement! o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard))) + } var txns []Tx for _, w := range dbs.W { @@ -153,7 +202,8 @@ type DBPerShard struct { // Easily see how many we have. Flatmap map[*DBShard]struct{} - types []txtype + types []txtype + hasRoaring bool txf *TxFactory @@ -163,6 +213,25 @@ type DBPerShard struct { useOpenList int } +// HasData returns true if the database has at least one key. +// For roaring it returns the number of fragments stored. +// The `which` argument is the index into the per.W slice. 0 for blue, 1 for green. +// If you pass 1, be sure you have a blue-green configuration. +func (per *DBPerShard) HasData(which int) (hasData bool, err error) { + // has to aggregate across all available DBShard for each index and shard. + + for v := range per.Flatmap { + hasData, err = v.W[which].HasData() + if err != nil { + return + } + if hasData { + return + } + } + return +} + func (per *DBPerShard) ListOpenString() (r string) { for v := range per.Flatmap { r += v.Path + " -> " + v.W[per.useOpenList].OpenListString() + "\n" @@ -173,12 +242,19 @@ func (per *DBPerShard) ListOpenString() (r string) { func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string) (d *DBPerShard) { useOpenList := 0 + hasRoaring := false + if types[0] == roaringTxn { + hasRoaring = true + } if len(types) == 2 { // blue-green, avoid the empty roaring Tx open list. // Prefer B's open list if neither is roaring. if types[0] == roaringTxn || types[1] != roaringTxn { useOpenList = 1 } + if types[1] == roaringTxn { + hasRoaring = true + } } d = &DBPerShard{ @@ -188,6 +264,7 @@ func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string) (d *DBPerS Flatmap: make(map[*DBShard]struct{}), txf: txf, useOpenList: useOpenList, + hasRoaring: hasRoaring, } return } @@ -239,28 +316,33 @@ func (per *DBPerShard) DeleteFieldFromStore(index, field, fieldPath string) (err func (per *DBPerShard) DeleteFragment(index, field, view string, shard uint64, frag *fragment) error { - dbs, err := per.GetDBShard(index, shard, nil) + idx := per.txf.holder.Index(index) + dbs, err := per.GetDBShard(index, shard, idx) panicOn(err) return dbs.DeleteFragment(index, field, view, shard, frag) } func (dbs *DBShard) DumpAll() { - + short := false + fmt.Printf("\n============= begin DumpAll dbs=%p index='%v', shard=%v ========\n", dbs, dbs.Index, int(dbs.Shard)) for i, ty := range dbs.types { _ = i tx, err := dbs.W[i].NewTx(!writable, "", Txo{Index: dbs.idx}) panicOn(err) defer tx.Rollback() - tx.Dump() + fmt.Printf("\n============= dumping dbs.W[%v] %v ========\n", i, ty) + tx.Dump(short) switch ty { case roaringTxn: case rbfTxn: case lmdbTxn: + case badgerTxn: default: panic(fmt.Sprintf("unknown txtyp: '%v'", ty)) } } + fmt.Printf("\n============= end of DumpAll index='%v', shard=%v ========\n", dbs.Index, int(dbs.Shard)) } func (per *DBPerShard) DumpAll() { @@ -285,6 +367,7 @@ func (per *DBPerShard) Path(index string, shard uint64) string { } func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *DBShard, err error) { + per.Mu.Lock() defer per.Mu.Unlock() @@ -296,8 +379,10 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs * per.dbh.Index[index] = dbi } dbs, ok = dbi.Shard[shard] + if dbs != nil && dbs.closed { + panic(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'", dbs)) + } if !ok { - dbs = &DBShard{ types: per.types, ParentDBIndex: dbi, @@ -307,6 +392,7 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs * idx: idx, per: per, useOpenList: per.useOpenList, + hasRoaring: per.hasRoaring, } dbi.Shard[shard] = dbs } @@ -320,11 +406,15 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs * registry = globalRbfDBReg case lmdbTxn: registry = globalLMDBReg + case badgerTxn: + registry = globalBadgerReg default: panic(fmt.Sprintf("unknown txtyp: '%v'", ty)) } w, err := registry.OpenDBWrapper(dbs.Path, DetectMemAccessPastTx) panicOn(err) + h := idx.Holder() + w.SetHolder(h) dbs.Open = true if w != nil && len(dbs.W) == 0 { per.Flatmap[dbs] = struct{}{} @@ -364,18 +454,39 @@ func (per *DBPerShard) Close() (err error) { return } -// requiredSuffix should be "-badgerdb" for badger, etc. +// DBPerShardGetShardsForIndex returns the indexes from the B (green) database if +// blue-green comparison is in use, rather than from the A (blue) database. func DBPerShardGetShardsForIndex(idx *Index, roaringViewPath string) (sliceOfShards []uint64, err error) { // follow the blueGreen convention of returning the answer for 'B' or // the last wrapper type. types := idx.holder.txf.Types() ty := types[len(types)-1] + return TypedDBPerShardGetLocalShardsForIndex(ty, idx, roaringViewPath) +} + +// if roaringViewPath is "" then for ty == roaringTxn we go to disk to discover +// all the view paths under idx for type ty. +func TypedDBPerShardGetLocalShardsForIndex(ty txtype, idx *Index, roaringViewPath string) (sliceOfShards []uint64, err error) { if ty == roaringTxn { rx := &RoaringTx{ Index: idx, } + if roaringViewPath == "" { + for _, field := range idx.Fields() { + for _, view := range field.views() { + sos, err := rx.SliceOfShards("", "", "", view.path) + if err != nil { + return nil, + errors.Wrap(err, fmt.Sprintf( + "TypedDBPerShardGetLocalShardsForIndex roaringTxn view.path='%v'", view.path)) + } + sliceOfShards = append(sliceOfShards, sos...) + } + } + return dedupShardSlice(sliceOfShards), nil + } return rx.SliceOfShards("", "", "", roaringViewPath) } requiredSuffix := ty.FileSuffix() @@ -435,3 +546,192 @@ func listDirUnderDir(root string, includeRoot bool, requiredSuffix string, ignor }) return } + +// populateBlueFromGreen prepares for a blue_green run at startup time. +// +// It is called at the end of Holder.Open(). This allows the application +// of blue-green checking to pilosa instances that +// were previously run only with a single (solo) backend. +// +// PRE: This operation requires, at its start, either: +// +// (1) an empty blue database -- this allows transitioning from +// a solo database to blue_green checking where the solo +// becomes the green; or +// +// (2) that the blue data, if present, be logically +// identical to the green data -- this allows one to restart +// a pilosa that was already running in blue_green mode +// and remain in blue_green mode. +// +// In either case, the goal to to finish populateBlueFromGreen() +// and have the exact same logical set of data in both backends. +// +// Why must the data be identical after Holder.Open() finishes? +// Otherwise subsequent blue-green checks have no hope of +// being accurate. +// +// The blue is the destination -- this is always types[0]. +// The green source is always types[1]. The mnemonic is blue_geen. +// The blue is first, so it is in types[0]. The green +// is second, in types[1]. For example, with PILOSA_TXSRC=lmdb_roaring +// we have lmdb as blue, and roaring as green. The contents of +// lmdb must be empty or exactly match roaring. If lmdb +// starts empty, it will be populated from roaring by +// populateBlueFromGreen(). +// +func (dbs *DBShard) populateBlueFromGreen() (err error) { + n := len(dbs.W) + if n != 2 { + panic(fmt.Sprintf("copyGreenToBlue did not find 2 open DBs: have %v", n)) + } + + dest := dbs.W[0] // blue + src := dbs.W[1] // green + + // copy all the key/container pairs. + // Since a shard is fairly small, we think one Tx will suffice. + + readtx, err := src.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard}) + panicOn(err) + defer readtx.Rollback() + + writetx, err := dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard}) + panicOn(err) + defer writetx.Rollback() + + for _, fld := range dbs.idx.Fields() { + field := fld.Name() + for _, vw := range fld.views() { + view := vw.name + citer, _, err := readtx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0) + if err != nil { + return errors.Wrap(err, "DBShard.copyGreenToBlue readtx.ContainerIterator") + } + + for citer.Next() { + ckey, rc := citer.Value() + err := writetx.PutContainer(dbs.Index, field, view, dbs.Shard, ckey, rc) + if err != nil { + citer.Close() + return errors.Wrap(err, "DBShard.copyGreenToBlue writetx.PutContainer") + } + } + citer.Close() + } + } + err = writetx.Commit() + if err != nil { + return errors.Wrap(err, "writetx.Commit()") + } + return nil +} + +// verifyBlueEqualsGreen checks that blue and green are identical. +func (dbs *DBShard) verifyBlueEqualsGreen(numCtVerified *int64) (err error) { + + n := len(dbs.W) + if n != 2 { + panic(fmt.Sprintf("verifyBlueEqualsGreen did not find 2 open DBs: have %v", n)) + } + + blue := dbs.W[0] + green := dbs.W[1] + + greentx, err := green.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard}) + panicOn(err) + defer greentx.Rollback() + + bluetx, err := blue.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard}) + panicOn(err) + defer bluetx.Rollback() + + for _, fld := range dbs.idx.Fields() { + field := fld.Name() + for _, vw := range fld.views() { + + view := vw.name + gCiter, _, err := greentx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0) + if err != nil { + return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen greentx.ContainerIterator") + } + + bCiter, _, err := bluetx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0) + if err != nil { + gCiter.Close() + if bCiter != nil { + bCiter.Close() + } + return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen bluetx.ContainerIterator") + } + + for gCiter.Next() { + greenCkey, greenc := gCiter.Value() + + if !bCiter.Next() { + bCiter.Close() + gCiter.Close() + return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ + "sees missing blue container at index: '%v' field: '%v' view: '%v' "+ + "shard: '%v' the greenCkey: '%v'", + dbs.Index, field, view, dbs.Shard, greenCkey)) + } + blueCkey, bluec := bCiter.Value() + + if blueCkey != greenCkey { + bCiter.Close() + gCiter.Close() + return fmt.Errorf("DBShard.verifyBlueEqualsGreen sees sequence-of-ckey "+ + "difference: blueCkey %v not equal to greenCkey %v at index: '%v' field: '%v' view: '%v' "+ + "shard: '%v'", + blueCkey, greenCkey, dbs.Index, field, view, dbs.Shard) + } + nGreen := greenc.N() + nBlue := bluec.N() + if nBlue != nGreen { + bCiter.Close() + gCiter.Close() + return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ + "sees variation in blue at index: '%v' field: '%v' view: '%v' "+ + "shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v", + dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue)) + } + err = bluec.BitwiseCompare(greenc) + if err != nil { + bCiter.Close() + gCiter.Close() + return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ + "sees variation in blue at index: '%v' field: '%v' view: '%v' "+ + "shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v ; BitwiseCompare response: '%v'", + dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue, err)) + } + atomic.AddInt64(numCtVerified, 1) + + } + if bCiter.Next() { + blueCkey, _ := bCiter.Value() + bCiter.Close() + gCiter.Close() + return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ + "sees extra blue container (not present in green) at index: '%v' field: '%v' view: '%v' "+ + "shard: '%v' the ckey: '%v'", + dbs.Index, field, view, dbs.Shard, blueCkey)) + } + bCiter.Close() + gCiter.Close() + } + } + + return nil +} + +func dedupShardSlice(sos []uint64) (r []uint64) { + m := make(map[uint64]struct{}) + for _, s := range sos { + m[s] = struct{}{} + } + for k := range m { + r = append(r, k) + } + return +} diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go index 96b7602a1..175c24f65 100644 --- a/dbshard_internal_test.go +++ b/dbshard_internal_test.go @@ -54,7 +54,7 @@ func TestShardPerDB_SetBit(t *testing.T) { if err != nil { t.Fatal(err) } - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() if n := f.mustRow(tx, 120).Count(); n != 2 { @@ -72,12 +72,13 @@ func Test_DBPerShard_GetShardsForIndex(t *testing.T) { orig := os.Getenv("PILOSA_TXSRC") defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests! - for _, src := range []string{"lmdb", "roaring", "rbf"} { - makeSampleRoaringDir(tmpdir, src) + for _, src := range []string{"lmdb", "roaring", "badger", "rbf"} { + makeSampleRoaringDir(tmpdir, src, 0) os.Setenv("PILOSA_TXSRC", src) // must make Holder AFTER setting src. holder := NewHolder(tmpdir, nil) + idx, err := NewIndex(holder, tmpdir, "rick") panicOn(err) estd := "rick/_exists/views/standard" @@ -100,6 +101,7 @@ func Test_DBPerShard_GetShardsForIndex(t *testing.T) { } } } + holder.Close() } } @@ -135,36 +137,62 @@ rick/_exists/views/standard/fragments/219 rick/_exists/views/standard/fragments/223 `, "lmdb": ` -rick/0219-lmdb/data.mdb -rick/0219-lmdb/lock.mdb -rick/0093-lmdb/data.mdb -rick/0093-lmdb/lock.mdb -rick/0223-lmdb/data.mdb -rick/0223-lmdb/lock.mdb -rick/0215-lmdb/data.mdb -rick/0215-lmdb/lock.mdb -rick/0217-lmdb/data.mdb -rick/0217-lmdb/lock.mdb -rick/0221-lmdb/data.mdb -rick/0221-lmdb/lock.mdb +rick/0219-lmdb@/data.mdb +rick/0219-lmdb@/lock.mdb +rick/0093-lmdb@/data.mdb +rick/0093-lmdb@/lock.mdb +rick/0223-lmdb@/data.mdb +rick/0223-lmdb@/lock.mdb +rick/0215-lmdb@/data.mdb +rick/0215-lmdb@/lock.mdb +rick/0217-lmdb@/data.mdb +rick/0217-lmdb@/lock.mdb +rick/0221-lmdb@/data.mdb +rick/0221-lmdb@/lock.mdb +`, + "badger": ` +rick/0219-badgerdb@/000000.vlog +rick/0219-badgerdb@/KEYREGISTRY +rick/0219-badgerdb@/MANIFEST +rick/0219-badgerdb@/LOCK +rick/0221-badgerdb@/000000.vlog +rick/0221-badgerdb@/KEYREGISTRY +rick/0221-badgerdb@/MANIFEST +rick/0221-badgerdb@/LOCK +rick/0223-badgerdb@/000000.vlog +rick/0223-badgerdb@/KEYREGISTRY +rick/0223-badgerdb@/MANIFEST +rick/0223-badgerdb@/LOCK +rick/0093-badgerdb@/000000.vlog +rick/0093-badgerdb@/KEYREGISTRY +rick/0093-badgerdb@/MANIFEST +rick/0093-badgerdb@/LOCK +rick/0217-badgerdb@/000000.vlog +rick/0217-badgerdb@/KEYREGISTRY +rick/0217-badgerdb@/MANIFEST +rick/0217-badgerdb@/LOCK +rick/0215-badgerdb@/000000.vlog +rick/0215-badgerdb@/KEYREGISTRY +rick/0215-badgerdb@/MANIFEST +rick/0215-badgerdb@/LOCK `, "rbf": ` -rick/0223-rbfdb/wal/0000000000000001.wal -rick/0223-rbfdb/data -rick/0093-rbfdb/wal/0000000000000001.wal -rick/0093-rbfdb/data -rick/0217-rbfdb/wal/0000000000000001.wal -rick/0217-rbfdb/data -rick/0215-rbfdb/wal/0000000000000001.wal -rick/0215-rbfdb/data -rick/0221-rbfdb/wal/0000000000000001.wal -rick/0221-rbfdb/data -rick/0219-rbfdb/wal/0000000000000001.wal -rick/0219-rbfdb/data +rick/0223-rbfdb@/wal/0000000000000001.wal +rick/0223-rbfdb@/data +rick/0093-rbfdb@/wal/0000000000000001.wal +rick/0093-rbfdb@/data +rick/0217-rbfdb@/wal/0000000000000001.wal +rick/0217-rbfdb@/data +rick/0215-rbfdb@/wal/0000000000000001.wal +rick/0215-rbfdb@/data +rick/0221-rbfdb@/wal/0000000000000001.wal +rick/0221-rbfdb@/data +rick/0219-rbfdb@/wal/0000000000000001.wal +rick/0219-rbfdb@/data `, } -func makeSampleRoaringDir(root, txsrc string) { +func makeSampleRoaringDir(root, txsrc string, minBytes int) { fns := strings.Split(sampleRoaringDirList[txsrc], "\n") for _, fn := range fns { if fn == "" { @@ -174,6 +202,10 @@ func makeSampleRoaringDir(root, txsrc string) { panicOn(os.MkdirAll(path, 0755)) fd, err := os.Create(root + sep + fn) panicOn(err) + if minBytes > 0 { + _, err := fd.Write(make([]byte, minBytes)) + panicOn(err) + } fd.Close() } } diff --git a/dbshard_test.go b/dbshard_test.go index 380412d8d..d69aa7943 100644 --- a/dbshard_test.go +++ b/dbshard_test.go @@ -18,9 +18,7 @@ import ( "context" "fmt" "os" - "path/filepath" "reflect" - "strings" "testing" "github.com/pilosa/pilosa/v2" @@ -39,7 +37,8 @@ func skipForNonLMDB(t *testing.T) { } } -var _ = skipForNonLMDB // happy linter +var _ = skipForNonLMDB // happy linter +var _ = skipForNonBadger // happy linter func skipForNonBadger(t *testing.T) { src := os.Getenv("PILOSA_TXSRC") @@ -50,7 +49,7 @@ func skipForNonBadger(t *testing.T) { // Can't write it all to one shard like we do (did). func Test_DBPerShard_multiple_shards_used(t *testing.T) { - skipForNonBadger(t) + skipForNonLMDB(t) c := test.MustRunCluster(t, 1) defer c.Close() hldr := c.GetHolder(0) @@ -62,8 +61,8 @@ func Test_DBPerShard_multiple_shards_used(t *testing.T) { hldr.SetBit(index, "general", 11, 2) hldr.SetBit(index, "general", 11, ShardWidth+2) - //tx_suffix := "-lmdb" - tx_suffix := "-badgerdb" + types := pilosa.MustTxsrcToTxtype("lmdb") + tx_suffix := types[0].FileSuffix() root := hldr.Path() + sep + index shards := []string{"0000", "0001", "0002"} pathShard := []string{} @@ -75,7 +74,7 @@ func Test_DBPerShard_multiple_shards_used(t *testing.T) { if !DirExists(pathShard[i]) { panic(fmt.Sprintf("no shard made for pathShard[%v]='%v'", i, pathShard[i])) } - sz, err := DiskUse(pathShard[i], "") + sz, err := pilosa.DiskUse(pathShard[i], "") panicOn(err) if sz < 100 { @@ -90,28 +89,6 @@ func Test_DBPerShard_multiple_shards_used(t *testing.T) { } } -func DiskUse(root string, requiredSuffix string) (tot int, err error) { - if !DirExists(root) { - return -1, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root) - } - - err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if info == nil { - panic(fmt.Sprintf("info was nil for path = '%v'", path)) - } - if info.IsDir() { - // skip directories. - } else { - sz := info.Size() - if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) { - tot += int(sz) - } - } - return nil - }) - return -} - func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) { c := test.MustRunCluster(t, 1, diff --git a/diagnostics.go b/diagnostics.go index b3ab1dd1f..9c380c1b8 100644 --- a/diagnostics.go +++ b/diagnostics.go @@ -229,7 +229,7 @@ func (d *diagnosticsCollector) EnrichWithSchemaProperties() { timeQuantumEnabled := false for _, index := range d.server.holder.Indexes() { - numShards += index.AvailableShards().Count() + numShards += index.AvailableShards(includeRemote).Count() numIndexes++ for _, field := range index.Fields() { numFields++ diff --git a/executor.go b/executor.go index 3b5062fe2..4ef7e90a4 100644 --- a/executor.go +++ b/executor.go @@ -506,7 +506,7 @@ func (e *executor) execute(ctx context.Context, qcx *Qcx, index string, q *pql.Q if idx == nil { return nil, newNotFoundError(ErrIndexNotFound, index) } - shards = idx.AvailableShards().Slice() + shards = idx.AvailableShards(includeRemote).Slice() if len(shards) == 0 { shards = []uint64{0} } @@ -780,7 +780,7 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p if idx == nil { return nil, newNotFoundError(ErrIndexNotFound, index) } - shards = idx.AvailableShards().Slice() + shards = idx.AvailableShards(includeRemote).Slice() if len(shards) == 0 { shards = []uint64{0} } @@ -4808,7 +4808,7 @@ func (e *executor) translateCalls(ctx context.Context, defaultIndexName string, // Perform a separate batch translation for each separate index used. keyMaps := make(map[string]map[string]uint64) for indexName, keySet := range keySets { - idx := e.Holder.indexes[indexName] + idx := e.Holder.Index(indexName) if idx == nil { return fmt.Errorf("cannot find index %q", indexName) } @@ -4851,8 +4851,8 @@ func (e *executor) collectCallKeySets(ctx context.Context, indexName string, c * // Collect foreign index keys. if fieldName != "" { - idx, exists := e.Holder.indexes[indexName] - if !exists { + idx := e.Holder.Index(indexName) + if idx == nil { return newNotFoundError(ErrIndexNotFound, indexName) } if field := idx.Field(fieldName); field != nil && field.ForeignIndex() != "" { @@ -4899,8 +4899,8 @@ func (e *executor) translateCall(ctx context.Context, indexName string, c *pql.C // Translate column key. colKey, rowKey, fieldName := c.TranslateInfo(columnLabel, rowLabel) - idx, exists := e.Holder.indexes[indexName] - if !exists { + idx := e.Holder.Index(indexName) + if idx == nil { return newNotFoundError(ErrIndexNotFound, indexName) } if idx.Keys() { diff --git a/executor_internal_test.go b/executor_internal_test.go index cd7347869..622037032 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -155,7 +155,7 @@ func TestExecutor_TranslateRowsOnBool(t *testing.T) { } shard := uint64(0) - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) defer tx.Rollback() fb, errb := idx.CreateField("b", OptFieldTypeBool()) diff --git a/executor_test.go b/executor_test.go index eb9dad815..e79a0e301 100644 --- a/executor_test.go +++ b/executor_test.go @@ -936,7 +936,7 @@ func TestExecutor_Execute_SetValue(t *testing.T) { // Obtain transaction. idx := index.Index shard := uint64(0) - tx := idx.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idx, Shard: shard}) + tx := idx.Txf().NewTx(pilosa.Txo{Write: !writable, Index: idx, Shard: shard}) defer tx.Rollback() f := hldr.Field("i", "f") diff --git a/field.go b/field.go index 2073bf8ac..9e0b13ad2 100644 --- a/field.go +++ b/field.go @@ -412,11 +412,31 @@ func (f *Field) TranslateStore() TranslateStore { func (f *Field) RowAttrStore() AttrStore { return f.rowAttrStore } // AvailableShards returns a bitmap of shards that contain data. -func (f *Field) AvailableShards() *roaring.Bitmap { +func (f *Field) AvailableShards(localOnly bool) *roaring.Bitmap { f.mu.RLock() defer f.mu.RUnlock() - b := f.remoteAvailableShards.Clone() + var b *roaring.Bitmap + if localOnly { + b = roaring.NewBitmap() + } else { + b = f.remoteAvailableShards.Clone() + } + for _, view := range f.viewMap { + //b.Union(view.availableShards()) + b.UnionInPlace(view.availableShards()) + } + return b +} + +// LocalAvailableShards returns a bitmap of shards that contain data, but +// only from the local node. This prevents txfactory from making +// db-per-shard for remote shards. +func (f *Field) LocalAvailableShards() *roaring.Bitmap { + f.mu.RLock() + defer f.mu.RUnlock() + + b := roaring.NewBitmap() for _, view := range f.viewMap { //b.Union(view.availableShards()) b.UnionInPlace(view.availableShards()) @@ -687,7 +707,7 @@ func (f *Field) applyTranslateStore() error { // In the case where the field has a foreign index, set // the usesKeys value accordingly. if foreignIndexName := f.ForeignIndex(); foreignIndexName != "" { - if foreignIndex := f.holder.indexes[foreignIndexName]; foreignIndex != nil { + if foreignIndex := f.holder.Index(foreignIndexName); foreignIndex != nil { f.usesKeys = foreignIndex.Keys() } } @@ -1483,7 +1503,7 @@ func (f *Field) SetValue(tx Tx, columnID uint64, value int64) (changed bool, err if view.idx == nil { panic("view.idx should not be nil") } - view.holder.addIndexFromField(view.idx) + view.holder.addIndex(view.idx) return view.setValue(tx, columnID, bsig.BitDepth, baseValue) } @@ -1571,7 +1591,7 @@ func (f *Field) MinForShard(tx Tx, shard uint64, filter *Row) (ValCount, error) var localTx Tx if NilInside(tx) { - localTx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: fragment, Shard: fragment.shard}) + localTx = f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: fragment, Shard: fragment.shard}) defer localTx.Rollback() } else { localTx = tx diff --git a/field_internal_test.go b/field_internal_test.go index 1bb96c617..ad0f5faaf 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -208,6 +208,8 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField { t.Fatal(err) } h := NewHolder(path, nil) + panicOn(h.Open()) + idx, err := h.CreateIndex("i", IndexOptions{}) if err != nil { panic(err) @@ -232,7 +234,7 @@ func OpenField(t *testing.T, opts FieldOption) *TestField { // Close closes the field and removes the underlying data. func (f *TestField) Close() error { if f.idx != nil { - panicOn(f.idx.Txf.CloseIndex(f.idx)) + panicOn(f.idx.holder.txf.CloseIndex(f.idx)) } defer os.RemoveAll(f.Path()) return f.Field.Close() @@ -245,7 +247,7 @@ func (f *TestField) Reopen() error { f.parent = nil return err } - if err := f.parent.Open(false); err != nil { + if err := f.parent.Open(); err != nil { f.parent = nil return err } @@ -318,7 +320,7 @@ func TestField_RowTime(t *testing.T) { defer f.Close() // Obtain transaction. - tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field, Shard: 0}) + tx := f.idx.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field, Shard: 0}) defer tx.Rollback() if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil { @@ -334,7 +336,7 @@ func TestField_RowTime(t *testing.T) { panicOn(tx.Commit()) // obtain 2nd transaction to read it back. - tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: 0}) + tx = f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: 0}) defer tx.Rollback() if r, err := f.RowTime(tx, 1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil { @@ -574,7 +576,7 @@ func TestBSIGroup_importValue(t *testing.T) { f := OpenField(t, OptFieldTypeInt(-100, 200)) defer f.Close() - qcx := f.idx.Txf.NewQcx() + qcx := f.idx.holder.txf.NewQcx() defer qcx.Abort() options := &ImportOptions{} @@ -620,7 +622,7 @@ func TestIntField_MinMaxForShard(t *testing.T) { f := OpenField(t, OptFieldTypeInt(-100, 200)) defer f.Close() - qcx := f.idx.Txf.NewQcx() + qcx := f.idx.holder.txf.NewQcx() defer qcx.Abort() options := &ImportOptions{} @@ -679,7 +681,7 @@ func TestIntField_MinMaxForShard(t *testing.T) { panicOn(qcx.Finish()) shard := uint64(0) - tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard}) + tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard}) // Rollback below manually, because we are in a loop. maxvc, err := f.MaxForShard(tx, shard, nil) @@ -705,6 +707,7 @@ func TestIntField_MinMaxForShard(t *testing.T) { // Ensure we get errors when they are expected. func TestDecimalField_MinMaxBoundaries(t *testing.T) { th := newTestHolder(t) + defer th.Close() for i, test := range []struct { scale int64 min pql.Decimal @@ -785,7 +788,7 @@ func TestDecimalField_MinMaxForShard(t *testing.T) { f := OpenField(t, OptFieldTypeDecimal(3)) defer f.Close() - qcx := f.idx.Txf.NewQcx() + qcx := f.idx.holder.txf.NewQcx() defer qcx.Abort() options := &ImportOptions{} @@ -843,7 +846,7 @@ func TestDecimalField_MinMaxForShard(t *testing.T) { } shard := uint64(0) - tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard}) + tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: shard}) defer tx.Rollback() maxvc, err := f.MaxForShard(tx, shard, nil) @@ -869,7 +872,7 @@ func TestBSIGroup_TxReopenDB(t *testing.T) { f := OpenField(t, OptFieldTypeInt(-100, 200)) defer f.Close() - qcx := f.idx.Txf.NewQcx() + qcx := f.idx.holder.txf.NewQcx() defer qcx.Abort() options := &ImportOptions{} diff --git a/field_test.go b/field_test.go index ea0c3e1e2..af58039a1 100644 --- a/field_test.go +++ b/field_test.go @@ -205,6 +205,8 @@ func TestField_NameValidation(t *testing.T) { } } +const includeRemote = false // for calls to Index.AvailableShards(localOnly bool) + // Ensure can update and delete available shards. func TestField_AvailableShards(t *testing.T) { idx := test.MustOpenIndex(t) @@ -223,7 +225,7 @@ func TestField_AvailableShards(t *testing.T) { t.Fatal(err) } else if _, err := f.SetBit(tx, 0, ShardWidth*2, nil); err != nil { t.Fatal(err) - } else if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" { + } else if diff := cmp.Diff(f.AvailableShards(includeRemote).Slice(), []uint64{0, 2}); diff != "" { t.Fatal(diff) } @@ -231,7 +233,7 @@ func TestField_AvailableShards(t *testing.T) { if err := f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)); err != nil { t.Fatalf("adding remote shards: %v", err) } - if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 1, 2, 4}); diff != "" { + if diff := cmp.Diff(f.AvailableShards(includeRemote).Slice(), []uint64{0, 1, 2, 4}); diff != "" { t.Fatal(diff) } @@ -242,7 +244,7 @@ func TestField_AvailableShards(t *testing.T) { t.Fatalf("removing shard %d: %v", i, err) } } - if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" { + if diff := cmp.Diff(f.AvailableShards(includeRemote).Slice(), []uint64{0, 2}); diff != "" { t.Fatal(diff) } } diff --git a/fragment.go b/fragment.go index cc8423e02..d248f4665 100644 --- a/fragment.go +++ b/fragment.go @@ -173,8 +173,9 @@ type fragment struct { // newFragment returns a new instance of Fragment. func newFragment(holder *Holder, path, index, field, view string, shard uint64, flags byte) *fragment { idx := holder.Index(index) + if idx == nil { - panic(fmt.Sprintf("holder=%#v but got nil idx back from holder!", holder)) + panic(fmt.Sprintf("got nil idx back for '%v' from holder!", index)) } f := &fragment{ path: path, @@ -478,7 +479,7 @@ func (f *fragment) openCache() error { return nil } - tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f, Shard: f.shard}) + tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Read in all rows by ID. @@ -996,7 +997,7 @@ func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint, value tx := txOrig if NilInside(tx) { - tx = f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Fragment: f, Shard: f.shard}) + tx = f.idx.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Fragment: f, Shard: f.shard}) defer func() { if err == nil { panicOn(tx.Commit()) @@ -1996,9 +1997,9 @@ func (f *fragment) Blocks() ([]FragmentBlock, error) { idx := f.holder.Index(f.index) if idx == nil { - panic(fmt.Sprintf("index was nil in fragment.Blocks(): f.index='%v'; f.holder.indexes='%#v'\n", f.index, f.holder.indexes)) + panic(fmt.Sprintf("index was nil in fragment.Blocks(): f.index='%v'\n", f.index)) } - tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // no Commit below, b/c is read-only. @@ -2083,7 +2084,7 @@ func (f *fragment) blockData(id int) (rowIDs, columnIDs []uint64, err error) { defer f.mu.Unlock() idx := f.holder.Index(f.index) - tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Shard: f.shard}) + tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: f.shard}) defer tx.Rollback() // readonly, so no Commit() @@ -2785,7 +2786,7 @@ func (f *fragment) WriteTo(w io.Writer) (n int64, err error) { // used in shipping the slices across the network for a resize. func (f *fragment) writeStorageToArchive(tw *tar.Writer) error { - tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Shard: f.shard}) + tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Shard: f.shard}) defer tx.Rollback() file, sz, err := tx.RoaringBitmapReader(f.index, f.field, f.view, f.shard, f.path) if err != nil { @@ -2859,7 +2860,7 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) { switch hdr.Name { case "data": idx := f.holder.Index(f.index) - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() if err := f.fillFragmentFromArchive(tx, tr); err != nil { return 0, errors.Wrap(err, "reading storage") @@ -3639,7 +3640,7 @@ func (s *fragmentSyncer) syncBlock(id int) error { } idx := f.holder.Index(f.index) - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: f.shard}) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: f.shard}) defer tx.Rollback() // Merge blocks together. diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 1597b93d1..394a8cabf 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -81,7 +81,7 @@ func TestFragment_SetBit(t *testing.T) { if err != nil { t.Fatal(err) } - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() if n := f.mustRow(tx, 120).Count(); n != 2 { @@ -114,7 +114,7 @@ func TestFragment_ClearBit(t *testing.T) { // In that spirit, we will check that the Tx Commit is visible afterwards. panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Close and reopen the fragment & verify the data. @@ -199,7 +199,7 @@ func TestFragment_ClearRow(t *testing.T) { t.Fatalf("unexpected count: %d", n) } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Close and reopen the fragment & verify the data. @@ -246,7 +246,7 @@ func TestFragment_SetRow(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Verify data on row. @@ -259,7 +259,7 @@ func TestFragment_SetRow(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Close and reopen the fragment & verify the data. @@ -304,7 +304,7 @@ func TestFragment_SetValue(t *testing.T) { if err := tx.Commit(); err != nil { t.Fatal(err) } - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Read value. @@ -356,7 +356,7 @@ func TestFragment_SetValue(t *testing.T) { if err := tx.Commit(); err != nil { t.Fatal(err) } - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() if value, exists, err := f.value(tx, 100, 16); err != nil { @@ -401,7 +401,7 @@ func TestFragment_SetValue(t *testing.T) { if err := tx.Commit(); err != nil { t.Fatal(err) } - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() if value, exists, err := f.value(tx, 100, 16); err != nil { @@ -478,7 +478,7 @@ func TestFragment_SetValue(t *testing.T) { if err := tx.Commit(); err != nil { t.Fatal(err) } - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Ensure values are set. @@ -525,7 +525,7 @@ func TestFragment_Sum(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() t.Run("NoFilter", func(t *testing.T) { @@ -549,7 +549,7 @@ func TestFragment_Sum(t *testing.T) { }) panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // verify that clearValue clears values @@ -558,7 +558,7 @@ func TestFragment_Sum(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() t.Run("ClearValue", func(t *testing.T) { @@ -599,7 +599,7 @@ func TestFragment_MinMax(t *testing.T) { panicOn(tx.Commit()) // the new tx is shared by Min/Max below. - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() t.Run("Min", func(t *testing.T) { @@ -1197,7 +1197,7 @@ func TestFragment_Snapshot(t *testing.T) { t.Fatal(err) } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Snapshot bitmap and verify data. @@ -1290,7 +1290,7 @@ func TestFragment_Top_Filter(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Retrieve top rows. @@ -1467,7 +1467,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { defer f.Clean(t) // Obtain transaction. - tx := index.Txf.NewTx(Txo{Write: writable, Index: index, Fragment: f, Shard: f.shard}) + tx := index.holder.txf.NewTx(Txo{Write: writable, Index: index, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Set bits on various rows. @@ -1546,7 +1546,7 @@ func TestFragment_Blocks(t *testing.T) { } prev = blocks - tx = idx.Txf.NewTx(Txo{Write: true, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: true, Index: idx, Fragment: f, Shard: f.shard}) // Set bit on different row. if _, err := f.setBit(tx, 20, 0); err != nil { t.Fatal(err) @@ -1561,7 +1561,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different column. - tx = idx.Txf.NewTx(Txo{Write: true, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: true, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() if _, err := f.setBit(tx, 20, 100); err != nil { t.Fatal(err) @@ -1658,7 +1658,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { } // Obtain transaction. - tx := index.Txf.NewTx(Txo{Write: writable, Index: index, Fragment: f, Shard: f.shard}) + tx := index.holder.txf.NewTx(Txo{Write: writable, Index: index, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Set bits on the fragment. @@ -1669,7 +1669,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { } panicOn(tx.Commit()) - tx = index.Txf.NewTx(Txo{Write: !writable, Index: index, Fragment: f, Shard: f.shard}) + tx = index.holder.txf.NewTx(Txo{Write: !writable, Index: index, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Verify correct cache type and size. @@ -1808,7 +1808,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Snapshot to disk before benchmarking. @@ -2149,7 +2149,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Check for expected results. @@ -2161,7 +2161,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Clear import. @@ -2171,7 +2171,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Check for expected results. @@ -2199,12 +2199,12 @@ func TestFragment_ConcurrentImport(t *testing.T) { eg := errgroup.Group{} eg.Go(func() error { - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard}) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard}) defer func() { panicOn(tx.Commit()) }() return f.bulkImportStandard(tx, []uint64{1, 2}, []uint64{1, 2}, &ImportOptions{}) }) eg.Go(func() error { - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard}) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard}) defer func() { panicOn(tx.Commit()) }() return f.bulkImportStandard(tx, []uint64{3, 4}, []uint64{3, 4}, &ImportOptions{}) }) @@ -2428,7 +2428,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Check for expected results. @@ -2440,7 +2440,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Clear import. @@ -2450,7 +2450,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Check for expected results. @@ -2678,7 +2678,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Check for expected results. @@ -2690,7 +2690,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Clear import. @@ -2700,7 +2700,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Check for expected results. @@ -2763,7 +2763,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) { i++ } - tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() if err := f.bulkImport(tx, rows, cols, options); err != nil { @@ -3137,6 +3137,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { fi.Close() h := NewHolder(fi.Name(), nil) + panicOn(h.Open()) idx, err := h.CreateIndex("i", IndexOptions{}) panicOn(err) @@ -3147,7 +3148,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { } // Obtain transaction. - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() copy(rows, rowsOrig) @@ -3195,7 +3196,7 @@ func BenchmarkImportRoaringIntoLargeFragment(b *testing.B) { f := newFragment(th, fi.Name(), "i", "f", viewStandard, 0, 0) defer f.Clean(b) - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() err = f.Open() @@ -3432,6 +3433,7 @@ func mustOpenBSIFragment(tb testing.TB, index, field, view string, shard uint64) func newTestHolder(tb testing.TB) *Holder { path, _ := testhook.TempDirInDir(tb, *TempDir, "holder-dir") h := NewHolder(path, nil) + panicOn(h.Open()) //h.SnapshotQueue = newSnapshotQueue(1, 1, nil) return h } @@ -3446,7 +3448,7 @@ func fragTestMustOpenIndex(index string, holder *Holder, opt IndexOptions) *Inde idx.keys = opt.Keys idx.trackExistence = opt.TrackExistence - if err := idx.Open(false); err != nil { + if err := idx.Open(); err != nil { panic(err) } return idx @@ -3472,10 +3474,10 @@ func mustOpenFragmentFlags(tb testing.TB, index, field, view string, shard uint6 fragPath := fragDir + fmt.Sprintf("%v", shard) f := newFragment(th, fragPath, index, field, view, shard, flags) - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard}) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard}) testhook.Cleanup(tb, func() { tx.Rollback() - panicOn(idx.Txf.CloseIndex(idx)) + panicOn(idx.holder.txf.CloseIndex(idx)) }) f.CacheType = cacheType @@ -3582,7 +3584,7 @@ func TestFragment_RowsIteration(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() ids, err := f.rows(context.Background(), tx, 0) @@ -4010,7 +4012,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { f.mustSetBits(tx, 3, 0) panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() iter, err := f.rowIterator(tx, false) @@ -4058,7 +4060,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { f.mustSetBits(tx, 7, 0) panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() iter, err := f.rowIterator(tx, false) @@ -4106,7 +4108,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { f.mustSetBits(tx, 3, 0) panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() iter, err := f.rowIterator(tx, true) @@ -4143,7 +4145,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { f.mustSetBits(tx, 7, 0) panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() iter, err := f.rowIterator(tx, true) @@ -4166,7 +4168,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) { t.Errorf("should have wrapped") } if !reflect.DeepEqual(row.Columns(), []uint64{0}) { - t.Fatalf("got wrong columns back on iteration %d - should just be 0 but %v", i, row.Columns()) + t.Fatalf("got wrong columns back on iteration %d - should just be 0 but got %v", i, row.Columns()) } } }) @@ -4392,7 +4394,7 @@ func TestFragmentBSIUnsigned(t *testing.T) { t.Run("<", func(t *testing.T) { - tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) + tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) defer tx.Rollback() for i := minCheck; i < maxCheck; i++ { @@ -4416,7 +4418,7 @@ func TestFragmentBSIUnsigned(t *testing.T) { }) t.Run("<=", func(t *testing.T) { - tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) + tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) defer tx.Rollback() for i := minCheck; i < maxCheck; i++ { @@ -4440,7 +4442,7 @@ func TestFragmentBSIUnsigned(t *testing.T) { }) t.Run(">", func(t *testing.T) { - tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) + tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) defer tx.Rollback() for i := minCheck; i < maxCheck; i++ { @@ -4463,7 +4465,7 @@ func TestFragmentBSIUnsigned(t *testing.T) { } }) t.Run(">=", func(t *testing.T) { - tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) + tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) defer tx.Rollback() for i := minCheck; i < maxCheck; i++ { @@ -4487,7 +4489,7 @@ func TestFragmentBSIUnsigned(t *testing.T) { }) t.Run("Range", func(t *testing.T) { - tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) + tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) defer tx.Rollback() for i := minCheck; i < maxCheck; i++ { @@ -4523,7 +4525,7 @@ func TestFragmentBSIUnsigned(t *testing.T) { }) t.Run("==", func(t *testing.T) { - tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) + tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: shard}) defer tx.Rollback() for i := minCheck; i < maxCheck; i++ { @@ -4563,7 +4565,7 @@ func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Generate a list of columns. @@ -4726,7 +4728,7 @@ func TestFragmentBSISigned(t *testing.T) { } panicOn(tx.Commit()) - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() // Generate a list of columns. @@ -4926,7 +4928,7 @@ func TestImportClearRestart(t *testing.T) { if err != nil { t.Fatalf("initial small import: %v", err) } - if idx.Txf.TxType() == RoaringTxn { + if idx.holder.txf.TxType() == RoaringTxn { if expOpN <= maxOpN && f.opN != expOpN { t.Errorf("unexpected opN - %d is not %d", f.opN, expOpN) } @@ -4940,13 +4942,13 @@ func TestImportClearRestart(t *testing.T) { panicOn(tx.Commit()) err = f.Open() - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() if err != nil { t.Fatalf("reopening fragment: %v", err) } - if idx.Txf.TxType() == RoaringTxn { + if idx.holder.txf.TxType() == RoaringTxn { if expOpN <= maxOpN && f.opN != expOpN { t.Errorf("unexpected opN after close/open %d is not %d", f.opN, expOpN) } @@ -4970,7 +4972,7 @@ func TestImportClearRestart(t *testing.T) { panicOn(tx.Commit()) // match the f.closeStorage which overlaps the f2 creation. - tx2 := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f2, Shard: f2.shard}) + tx2 := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f2, Shard: f2.shard}) defer tx2.Rollback() err = f.closeStorage() @@ -4983,7 +4985,7 @@ func TestImportClearRestart(t *testing.T) { t.Fatalf("opening new fragment: %v", err) } - if idx.Txf.TxType() == RoaringTxn { + if idx.holder.txf.TxType() == RoaringTxn { if expOpN <= maxOpN && f2.opN != expOpN { t.Errorf("unexpected opN after close/open %d is not %d", f2.opN, expOpN) } @@ -5016,7 +5018,7 @@ func TestImportClearRestart(t *testing.T) { f3.MaxOpN = maxOpN f3.CacheType = f.CacheType - tx3 := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f3, Shard: f3.shard}) + tx3 := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f3, Shard: f3.shard}) defer tx3.Rollback() err = f2.closeStorage() @@ -5072,7 +5074,7 @@ func TestImportValueConcurrent(t *testing.T) { // we will be making a new Tx each time, so we can rollback the default provided one. tx.Rollback() - types := idx.Txf.TxTypes() + types := idx.holder.txf.TxTypes() for _, ty := range types { switch ty { case roaringTxn: @@ -5090,7 +5092,7 @@ func TestImportValueConcurrent(t *testing.T) { for i := 0; i < 4; i++ { i := i eg.Go(func() error { - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() for j := uint64(0); j < 10; j++ { err := f.importValue(tx, []uint64{j}, []int64{int64(rand.Int63n(1000))}, 10, i%2 == 0) @@ -5138,7 +5140,7 @@ func TestImportMultipleValues(t *testing.T) { // probably too slow, would hit disk alot: //panicOn(tx.Commit()) - //tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard:f.shard, ShardSet:true}) + //tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard:f.shard, ShardSet:true}) //defer tx.Rollback() for i := range test.checkCols { @@ -5232,13 +5234,11 @@ func TestFragmentConcurrentReadWrite(t *testing.T) { f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) tx.Rollback() - // Obtain transaction, but don't start another b/c the - // two goroutines below need the same view. eg := &errgroup.Group{} eg.Go(func() error { - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) for i := uint64(0); i < 1000; i++ { _, err := f.setBit(tx, i%4, i) @@ -5251,7 +5251,7 @@ func TestFragmentConcurrentReadWrite(t *testing.T) { }) // need read-only Tx so as not to block on the writer finishing above. - tx = idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() acc := uint64(0) diff --git a/go.mod b/go.mod index 0c24f92df..4bcaa3810 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,8 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 + github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 + github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 github.com/glycerine/lmdb-go v1.9.32 github.com/go-ole/go-ole v1.2.4 // indirect github.com/gogo/protobuf v1.2.1 @@ -31,7 +33,6 @@ require ( github.com/satori/go.uuid v1.2.0 github.com/shirou/gopsutil v2.18.12+incompatible github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.4.0 @@ -45,7 +46,6 @@ require ( golang.org/x/text v0.3.3 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/grpc v1.28.0 - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect modernc.org/mathutil v1.0.0 modernc.org/strutil v1.0.0 vitess.io/vitess v3.0.0-rc.3.0.20190602171040-12bfde34629c+incompatible diff --git a/go.sum b/go.sum index 6a9cc69d2..39084cbd8 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWu github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= @@ -33,15 +35,25 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 h1:JBNM90aGLCiF9iJYvpvayMpYeW498v5ZDZqE2chqZ2A= +github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= @@ -76,6 +88,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= @@ -185,6 +199,7 @@ github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Ung github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -206,6 +221,7 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= @@ -214,6 +230,7 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -231,6 +248,7 @@ github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0= @@ -247,6 +265,7 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -288,9 +307,11 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/gossip/gossip.go b/gossip/gossip.go index 6e7ebb464..d7b6377e7 100644 --- a/gossip/gossip.go +++ b/gossip/gossip.go @@ -330,7 +330,7 @@ func (g *memberSet) LocalState(join bool) []byte { for _, f := range idx.Fields { availableShards := roaring.NewBitmap() if field, _ := g.papi.Field(context.Background(), idx.Name, f.Name); field != nil { - availableShards = field.AvailableShards() + availableShards = field.AvailableShards(false) } fs := &pilosa.FieldStatus{ diff --git a/holdbkg.go b/holdbkg.go new file mode 100644 index 000000000..1a2ef2e6c --- /dev/null +++ b/holdbkg.go @@ -0,0 +1,135 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "github.com/glycerine/idem" +) + +// holderBackgroundGoro avoids the deadlock versus race dilmena +// when creating a new index. The troubles were in having +// the field/view code try to inform the Holder of the indexes +// from a platoon of worker goroutines and tests; see index.go:273 inside +// Index.openFields(). There calling i.holder.addIndex(i) instead +// of locking cleans things up alot. There's no need anymore +// to track which goroutines are holding the Holder's +// mu lock. +type holderBackgroundGoro struct { + h *Holder + halt *idem.Halter + reqIndexCh chan *indexReq + setIdxCh chan *Index + getAllCh chan *indexReq + delIdxCh chan string +} + +// indexReq is used to ask the holderBackgrounGoro +// for index information. +type indexReq struct { + // basic request to map an index name to *Index + index string + idx *Index + + // double duty as a getAll indexes request + getAll bool + all []*Index + + done chan struct{} +} + +func newIndexReq(index string) *indexReq { + return &indexReq{ + index: index, + done: make(chan struct{}), + } +} + +func (b *holderBackgroundGoro) index(index string) *Index { + if b == nil { + // utils_internal_test.go:448 from + // TestCluster_ResizeStates/Multiple_nodes,_with_data + // wants to pass a nil b/nil Holder. sigh. don't panic. + return nil + } + req := newIndexReq(index) + b.reqIndexCh <- req + <-req.done + return req.idx +} + +func (b *holderBackgroundGoro) isClosed() bool { + select { + case <-b.halt.Done.Chan: + return true + default: + return false + } +} + +func (b *holderBackgroundGoro) start() { + go func() { + defer b.halt.Done.Close() + for { + select { + case <-b.halt.ReqStop.Chan: + return + + case r := <-b.reqIndexCh: + r.idx = b.h.indexesOwnedByBkgr[r.index] + close(r.done) + + case idx := <-b.setIdxCh: + b.h.indexesOwnedByBkgr[idx.Name()] = idx + // atomic operation, client doesn't need to wait on done, + // so don't take the time to do another channel, just leave done open. + + case target := <-b.delIdxCh: + delete(b.h.indexesOwnedByBkgr, target) + // atomic operation, client doesn't need to wait on done, + // so don't take the time to do another channel, just leave done open. + + case r := <-b.getAllCh: + r.all = make([]*Index, 0, len(b.h.indexesOwnedByBkgr)) + for _, index := range b.h.indexesOwnedByBkgr { + r.all = append(r.all, index) + } + close(r.done) + } + } + }() +} + +func (h *Holder) stopBkgr() { + h.mu.Lock() + defer h.mu.Unlock() + if h.bkgr != nil { + h.bkgr.halt.ReqStop.Close() + <-h.bkgr.halt.Done.Chan + } +} + +func (h *Holder) newHolderBackgroundGoro() (b *holderBackgroundGoro) { + + b = &holderBackgroundGoro{ + h: h, + halt: idem.NewHalter(), + reqIndexCh: make(chan *indexReq), + setIdxCh: make(chan *Index), + getAllCh: make(chan *indexReq), + delIdxCh: make(chan string), + } + b.start() + return b +} diff --git a/holder.go b/holder.go index 700487683..6848c13ab 100644 --- a/holder.go +++ b/holder.go @@ -60,12 +60,12 @@ func init() { type Holder struct { mu sync.RWMutex + // our configuration + cfg *HolderConfig + // Partition count used by translation. partitionN int - // Indexes by name. - indexes map[string]*Index - // opened channel is closed once Open() completes. opened lockedChan @@ -116,6 +116,32 @@ type Holder struct { Auditor testhook.Auditor txf *TxFactory + + bkgr *holderBackgroundGoro + + // indexesOwnedByBkgr is owned by h.bkgr. Do not touch. + // Only the owning holdbkg.go goroutine should query or + // modify the indexesOwnedByBkgr map. indexesOwnedByBkgr replaces + // the old indexes map which was the source of deadlock vs + // race issues with a dedicated goroutine + // associated with the Holder. + // + // Normally this map would live inside the holderBackgroundGoro + // struct, but tests expect the map to + // persist cross Holder ReOpens(), so it still lives here -- + // since a ReOpen will kill and restart the goroutine and + // lose its state. + // + // Again, do not touch this directly. Use these methods instead: + // + // h.Index() // index name -> *Index + // h.Indexes() // copy of the full list of *Indexes + // h.addIndex() // add one *Index + // h.deleteIndex() // delete one *Index + // + indexesOwnedByBkgr map[string]*Index + + numCtBlueGreenVerified int64 } // HolderOpts holds information about the holder which other things might want @@ -209,6 +235,8 @@ func DefaultHolderConfig() *HolderConfig { } // NewHolder returns a new instance of Holder for the given path. +// It starts the bkgr background goroutine that provides +// exclusive access to the indexesOwnedByBkgr map. func NewHolder(path string, cfg *HolderConfig) *Holder { if cfg == nil { cfg = DefaultHolderConfig() @@ -222,7 +250,7 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { } h := &Holder{ - indexes: make(map[string]*Index), + cfg: cfg, closing: make(chan struct{}), opened: lockedChan{ch: make(chan struct{})}, @@ -245,7 +273,11 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { Auditor: NewAuditor(), path: path, + + indexesOwnedByBkgr: make(map[string]*Index), } + h.bkgr = h.newHolderBackgroundGoro() + txf, err := NewTxFactory(cfg.Txsrc, path, h) panicOn(err) h.txf = txf @@ -535,9 +567,24 @@ func (h *Holder) Inspect(ctx context.Context, req *InspectRequest) (*HolderInfo, // Open initializes the root data directory for the holder. func (h *Holder) Open() error { + h.opening = true defer func() { h.opening = false }() + if h.bkgr == nil || h.bkgr.isClosed() { + h.bkgr = h.newHolderBackgroundGoro() + } + + if h.txf == nil { + txf, err := NewTxFactory(h.cfg.Txsrc, h.path, h) + if err != nil { + return errors.Wrap(err, "Holder.Open NewTxFactory()") + } + h.txf = txf + } + + h.txf.blueGreenOffIfRunningBlueGreen() + // Reset closing in case Holder is being reopened. h.closing = make(chan struct{}) @@ -580,9 +627,7 @@ func (h *Holder) Open() error { continue } // Skip embedded db files too. - if strings.HasSuffix(fi.Name(), "-badgerdb") || - strings.HasSuffix(fi.Name(), "-lmdb") || - strings.HasSuffix(fi.Name(), "-rbfdb") { + if h.txf.IsTxDatabasePath(fi.Name()) { continue } @@ -598,22 +643,19 @@ func (h *Holder) Open() error { if h.isCoordinator() { index.createdAt = timestamp() - err = index.OpenWithTimestamp(false) + err = index.OpenWithTimestamp() } else { - err = index.Open(false) + err = index.Open() } if err != nil { - // FIXME: The holder shouldn't be responsible for closing these, probably. - _ = index.Txf.CloseDB() + _ = h.txf.Close() if err == ErrName { h.Logger.Printf("ERROR opening index: %s, err=%s", index.Name(), err) continue } return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err) } - h.mu.Lock() - h.indexes[index.Name()] = index - h.mu.Unlock() + h.addIndex(index) } // If any fields were opened before their foreign index @@ -631,7 +673,14 @@ func (h *Holder) Open() error { _ = testhook.Opened(h.Auditor, h, nil) + // under blue_green, we must sync blue from green before we turn on checking. + if err := h.txf.green2blue(h); err != nil { + return errors.Wrap(err, "Holder.Open h.txf.UpdateBlueFromGreen(h)") + } + h.txf.blueGreenOnIfRunningBlueGreen() + return nil + } // Activate runs the background tasks relevant to keeping a holder in a stable @@ -676,10 +725,14 @@ func (h *Holder) processForeignIndexFields() error { // Close closes all open fragments. func (h *Holder) Close() error { + defer h.stopBkgr() + if globalUseStatTx { fmt.Printf("%v\n", globalCallStats.report()) } - h.txf.blueGreenReg.Close() + if h.txf.blueGreenReg != nil { + h.txf.blueGreenReg.Close() + } h.Stats.Close() @@ -687,16 +740,17 @@ func (h *Holder) Close() error { close(h.closing) h.wg.Wait() - for _, index := range h.indexes { + for _, index := range h.Indexes() { if err := index.Close(); err != nil { return errors.Wrap(err, "closing index") } - if err := index.Txf.CloseDB(); err != nil { - return errors.Wrap(err, "index.Txf.CloseDB()") - } + } + if err := h.txf.Close(); err != nil { + return errors.Wrap(err, "holder.Txf.Close()") } // Reset opened in case Holder needs to be reopened. + h.txf = nil h.opened.mu.Lock() h.opened.ch = make(chan struct{}) h.opened.mu.Unlock() @@ -713,7 +767,7 @@ func (h *Holder) Close() error { func (h *Holder) NeedsSnapshot() bool { h.mu.RLock() defer h.mu.RUnlock() - for _, idx := range h.indexes { + for _, idx := range h.Indexes() { if idx.NeedsSnapshot() { return true } @@ -727,7 +781,7 @@ func (h *Holder) NeedsSnapshot() bool { func (h *Holder) HasData() (bool, error) { h.mu.RLock() defer h.mu.RUnlock() - if len(h.indexes) > 0 { + if len(h.Indexes()) > 0 { return true, nil } // Open path to read all index directories. @@ -771,7 +825,7 @@ func (h *Holder) hasV1TranslateKeysFile() (bool, error) { func (h *Holder) availableShardsByIndex() map[string]*roaring.Bitmap { m := make(map[string]*roaring.Bitmap) for _, index := range h.Indexes() { - m[index.Name()] = index.AvailableShards() + m[index.Name()] = index.AvailableShards(includeRemote) } return m } @@ -914,27 +968,26 @@ func (h *Holder) HolderPathFromIndexPath(indexPath, indexName string) string { } // Index returns the index by name. -func (h *Holder) Index(name string) *Index { - h.mu.RLock() - defer h.mu.RUnlock() - return h.index(name) -} - -func (h *Holder) index(name string) *Index { - return h.indexes[name] +func (h *Holder) Index(name string) (idx *Index) { + idx = h.bkgr.index(name) + return } // Indexes returns a list of all indexes in the holder. func (h *Holder) Indexes() []*Index { - h.mu.RLock() - a := make([]*Index, 0, len(h.indexes)) - for _, index := range h.indexes { - a = append(a, index) + // utils_internal_test.go:345 func (t *ClusterCluster) Close() error + // wants to close an un-Open()-ed Holder. + // So I guess we won't panic here. + if h.bkgr == nil || h.bkgr.isClosed() { + return nil } - h.mu.RUnlock() + req := newIndexReq("") + req.getAll = true + h.bkgr.getAllCh <- req + <-req.done - sort.Sort(indexSlice(a)) - return a + sort.Sort(indexSlice(req.all)) + return req.all } // CreateIndex creates an index. @@ -944,7 +997,7 @@ func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) { defer h.mu.Unlock() // Ensure index doesn't already exist. - if h.index(name) != nil { + if h.Index(name) != nil { return nil, newConflictError(ErrIndexExists) } return h.createIndex(name, opt) @@ -957,10 +1010,9 @@ func (h *Holder) CreateIndexIfNotExists(name string, opt IndexOptions) (*Index, defer h.mu.Unlock() // Return index if it exists. - if index := h.index(name); index != nil { + if index := h.Index(name); index != nil { return index, nil } - return h.createIndex(name, opt) } @@ -978,7 +1030,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { index.keys = opt.Keys index.trackExistence = opt.TrackExistence - if err = index.Open(true); err != nil { + if err = index.Open(); err != nil { return nil, errors.Wrap(err, "opening") } if err = index.saveMeta(); err != nil { @@ -986,7 +1038,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) { } // Update options. - h.indexes[index.Name()] = index + h.addIndex(index) // Since this is a new index, we need to kick off // its translation sync. @@ -1017,7 +1069,7 @@ func (h *Holder) DeleteIndex(name string) error { defer h.mu.Unlock() // Confirm index exists. - index := h.index(name) + index := h.Index(name) if index == nil { return newNotFoundError(ErrIndexNotFound, name) } @@ -1028,7 +1080,7 @@ func (h *Holder) DeleteIndex(name string) error { } // remove any backing store. - if err := index.Txf.DeleteIndex(name); err != nil { + if err := h.txf.DeleteIndex(name); err != nil { return errors.Wrap(err, "index.Txf.DeleteIndex") } @@ -1038,7 +1090,7 @@ func (h *Holder) DeleteIndex(name string) error { } // Remove reference. - delete(h.indexes, name) + h.deleteIndex(name) // I'm not sure if calling Reset() here is necessary // since closing the index stops its translation @@ -1046,6 +1098,10 @@ func (h *Holder) DeleteIndex(name string) error { return h.translationSyncer.Reset() } +func (h *Holder) deleteIndex(index string) { + h.bkgr.delIdxCh <- index +} + // Field returns the field for an index and name. func (h *Holder) Field(index, name string) *Field { idx := h.Index(index) @@ -1287,7 +1343,7 @@ func (s *holderSyncer) SyncHolder() error { return nil } - itr := s.Holder.Index(di.Name).AvailableShards().Iterator() + itr := s.Holder.Index(di.Name).AvailableShards(includeRemote).Iterator() itr.Seek(0) for shard, eof := itr.Next(); !eof; shard, eof = itr.Next() { // Ignore shards that this host doesn't own. @@ -1736,7 +1792,7 @@ func (c *holderCleaner) CleanHolder() error { } // Get the fragments that node is responsible for (based on hash(index, node)). - containedShards := c.Cluster.containsShards(index.Name(), index.AvailableShards(), c.Node) + containedShards := c.Cluster.containsShards(index.Name(), index.AvailableShards(includeRemote), c.Node) // Get the fragments registered in memory. for _, field := range index.Fields() { @@ -1785,33 +1841,28 @@ func uint64InSlice(i uint64, s []uint64) bool { // Process loops through a holder based on the Check functions in op, calling // the Process functions in op when indicated. func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) { - var indexNames, fieldNames, viewNames []string + var fieldNames, viewNames []string var fragNums []uint64 - h.mu.Lock() - for indexName := range h.indexes { - indexNames = append(indexNames, indexName) - } - h.mu.Unlock() - for _, indexName := range indexNames { + indexes := h.Indexes() + for _, idx := range indexes { if err = ctx.Err(); err != nil { return err } + if idx == nil { + continue + } + indexName := idx.name process, recurse := op.CheckIndex(indexName) if !process && !recurse { continue } - h.mu.Lock() - index := h.indexes[indexName] - h.mu.Unlock() - if index == nil { - continue - } + if err = ctx.Err(); err != nil { return err } if process { - err = op.ProcessIndex(index) + err = op.ProcessIndex(idx) if err != nil { return err } @@ -1820,22 +1871,22 @@ func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) { continue } fieldNames = fieldNames[:0] - index.mu.Lock() - for fieldName := range index.fields { + idx.mu.Lock() + for fieldName := range idx.fields { fieldNames = append(fieldNames, fieldName) } - index.mu.Unlock() + idx.mu.Unlock() for _, fieldName := range fieldNames { if err = ctx.Err(); err != nil { return err } - process, recurse := op.CheckField(indexName, fieldName) + process, recurse := op.CheckField(idx.name, fieldName) if !process && !recurse { continue } - index.mu.Lock() - field := index.fields[fieldName] - index.mu.Unlock() + idx.mu.Lock() + field := idx.fields[fieldName] + idx.mu.Unlock() if field == nil { continue } @@ -1913,23 +1964,32 @@ func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) { // used by Index.openFields(), enabling Tx / Txf by telling // the holder about its own indexes. -func (h *Holder) addIndexFromField(idx *Index) { - h.mu.Lock() - h.indexes[idx.Name()] = idx - h.mu.Unlock() -} - -func (h *Holder) unprotectedAddIndexFromField(idx *Index) { - h.indexes[idx.Name()] = idx +func (h *Holder) addIndex(idx *Index) { + if idx == nil { + panic("cannot pass nil to addIndex") + } + if h == nil { + panic("cannot call addIndex on nil Holder") + } + if h.bkgr == nil || h.bkgr.isClosed() { + // ugh. TestCluster_ResizeStates/Multiple_nodes,_with_data test + // from cluster_internal_test.go:799 + // via utils_internal_test.go:450 + // gets here, with the h.mu already held. + // so we cannot panic and complain or we mess up that test. + // But really, we should not be calling addIndex() on + // Holder that has not be Open()-ed. + h.mu.Lock() + h.bkgr = h.newHolderBackgroundGoro() + h.mu.Unlock() + } + h.bkgr.setIdxCh <- idx } func (h *Holder) DumpAllShards() { h.mu.RLock() defer h.mu.RUnlock() - for index, idx := range h.indexes { - fmt.Printf("dump of index '%v'\n", index) - idx.Txf.dbPerShard.DumpAll() - } + h.txf.dbPerShard.DumpAll() } func (h *Holder) Txf() *TxFactory { @@ -1941,5 +2001,38 @@ func (h *Holder) Txf() *TxFactory { // Begin starts a transaction on the holder. The index and shard // must be specified. func (h *Holder) BeginTx(writable bool, idx *Index, shard uint64) (Tx, error) { - return idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}), nil + return h.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}), nil +} + +func (h *Holder) NumContainersBlueGreenVerified() int { + return int(h.numCtBlueGreenVerified) +} + +func (h *Holder) HasRoaringData() (has bool, err error) { + + idxs := h.Indexes() + for _, idx := range idxs { + paths, err := listFilesUnderDir(idx.path, false, "", true) + if err != nil { + return false, errors.Wrap(err, "HasRoaringData listFilesUnderDir") + } + index := idx.name + + for _, relpath := range paths { + field, view, shard, err := fragmentSpecFromRoaringPath(relpath) + if err != nil { + continue // ignore .meta paths + } + abspath := idx.path + sep + relpath + + hasData, err := roaringFragmentHasData(abspath, index, field, view, shard) + if err != nil { + return false, errors.Wrap(err, "HasRoaringData roaringFragmentHasData") + } + if hasData { + return true, nil + } + } + } + return } diff --git a/holder_internal_test.go b/holder_internal_test.go index c7e9a0bcb..1db02a8b0 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -16,12 +16,15 @@ package pilosa import ( "context" + "fmt" "os" "testing" "github.com/pilosa/pilosa/v2/testhook" ) +var _ = fmt.Printf + type testHolderOperator struct { indexSeen, indexProcessed int fieldSeen, fieldProcessed int @@ -79,7 +82,7 @@ func makeHolder(tb testing.TB) (*Holder, string, error) { return nil, "", err } h := NewHolder(path, nil) - return h, path, nil + return h, path, h.Open() } func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) { @@ -89,21 +92,81 @@ func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID ui t.Fatalf("creating index: %v", err) } - shard := columnID / ShardWidth - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) - defer tx.Rollback() - f, err := idx.CreateFieldIfNotExists(field, OptFieldTypeDefault()) if err != nil { t.Fatalf("setting bit: %v", err) } - _, err = f.SetBit(tx, rowID, columnID, nil) + _, err = f.SetBit(nil, rowID, columnID, nil) if err != nil { t.Fatalf("setting bit: %v", err) } - if err := tx.Commit(); err != nil { - t.Fatal(err) +} + +func testMustHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) { + + //shard := columnID / ShardWidth + + // hmm... if its a new holder, meta data isn't there, so ask for it. + idx, err := h.CreateIndexIfNotExists(index, IndexOptions{}) + panicOn(err) + + f := idx.Field(field) + if f == nil { + t.Fatalf("no such field '%v'", field) } + + row, err := f.Row(nil, rowID) + if err != nil { + t.Fatalf("error getting field.Row(rowID=%v): %v", rowID, err) + } + + cols := row.Columns() + if len(cols) == 0 { + t.Fatalf("error getting field.Row().Columns(): empty columns, colID %v bit was not hot", columnID) + } + + for _, c := range cols { + if c == columnID { + return // ok, found it. + } + } + t.Fatalf("error getting field.Row().Columns(): colID %v bit was not hot", columnID) +} + +func testMustNotHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) { + if testHasBit(t, h, index, field, rowID, columnID) { + t.Fatalf("error, expected no bit but this bit was hot: index='%v', field='%v', rowID='%v', columnID='%v'", index, field, rowID, columnID) + } +} + +func testHasBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) bool { + + idx := h.Index(index) + if idx == nil { + return false // not even an index by this name. Obviously no hot bits either. + } + + f := idx.Field(field) + if f == nil { + return false + } + + row, err := f.Row(nil, rowID) + if err != nil { + return false + } + + cols := row.Columns() + if len(cols) == 0 { + return false + } + + for _, c := range cols { + if c == columnID { + return true // ok, found it. + } + } + return false } func TestHolderOperatorProcess(t *testing.T) { diff --git a/holder_test.go b/holder_test.go index b79ef71f6..7ccad46f1 100644 --- a/holder_test.go +++ b/holder_test.go @@ -180,7 +180,7 @@ func TestHolder_Open(t *testing.T) { } var shard uint64 - tx := idx.Txf.NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard}) + tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard}) defer tx.Rollback() if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { @@ -214,7 +214,7 @@ func TestHolder_Open(t *testing.T) { } var shard uint64 - tx := idx.Txf.NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard}) + tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard}) if err != nil { t.Fatal(err) } @@ -247,7 +247,7 @@ func TestHolder_Open(t *testing.T) { t.Fatal(err) } var shard uint64 - tx := idx.Txf.NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard}) + tx := idx.Txf().NewTx(pilosa.Txo{Write: writable, Index: idx, Shard: shard}) defer tx.Rollback() if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil { diff --git a/index.go b/index.go index 726dca7c7..f77aaf354 100644 --- a/index.go +++ b/index.go @@ -70,8 +70,6 @@ type Index struct { // Instantiates new translation stores OpenTranslateStore OpenTranslateStoreFunc - - Txf *TxFactory } // NewIndex returns an existing (but possibly empty) instance of @@ -105,19 +103,16 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) { translationSyncer: NopTranslationSyncer, OpenTranslateStore: OpenInMemTranslateStore, - - // the Txf should be shared across all holder. - Txf: holder.txf, } return idx, nil } func (i *Index) NewTx(txo Txo) Tx { - return i.Txf.NewTx(txo) + return i.holder.txf.NewTx(txo) } func (i *Index) NeedsSnapshot() bool { - return i.Txf.NeedsSnapshot() + return i.holder.txf.NeedsSnapshot() } // CreatedAt is an timestamp for a specific version of an index. @@ -174,12 +169,14 @@ func (i *Index) options() IndexOptions { } // Open opens and initializes the index. -func (i *Index) Open(haveHolderLock bool) error { return i.open(false, haveHolderLock) } +func (i *Index) Open() error { + return i.open(false) +} // OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields. -func (i *Index) OpenWithTimestamp(haveHolderLock bool) error { return i.open(true, haveHolderLock) } +func (i *Index) OpenWithTimestamp() error { return i.open(true) } -func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) { +func (i *Index) open(withTimestamp bool) (err error) { // Ensure the path exists. i.holder.Logger.Debugf("ensure index path exists: %s", i.path) if err := os.MkdirAll(i.path, 0777); err != nil { @@ -193,7 +190,7 @@ func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) { } i.holder.Logger.Debugf("open fields for index: %s", i.name) - if err := i.openFields(withTimestamp, haveHolderLock); err != nil { + if err := i.openFields(withTimestamp); err != nil { return errors.Wrap(err, "opening fields") } @@ -240,7 +237,7 @@ func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) { var indexQueue = make(chan struct{}, 8) // openFields opens and initializes the fields inside the index. -func (i *Index) openFields(withTimestamp, haveHolderLock bool) error { +func (i *Index) openFields(withTimestamp bool) error { f, err := os.Open(i.path) if err != nil { return errors.Wrap(err, "opening directory") @@ -273,27 +270,8 @@ fileLoop: mu.Lock() - // i.holder needs to know about its index i for the Txf to work. - // - // We face either a deadlock or a race here. - // - // We get a deadlock in TestIndex_CreateField/"BSIFields"/"OK" - // if we call addIndexFromField, because in that test - // we get here while already holding i.holder.mu. - // - // On the other had, we get races on other tests - // such as TestExecutor_Execute_Existence/Row - // if we call unprotectedAddIndexFromField which does - // not lock i.holder.mu. - // - // The resolution was to have the goroutines that are holding - // the lock already tell us. That is the haveHolderLock - // argument. - if haveHolderLock { - i.holder.unprotectedAddIndexFromField(i) - } else { - i.holder.addIndexFromField(i) - } + // goroutine safe + i.holder.addIndex(i) fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) if withTimestamp { @@ -397,7 +375,7 @@ func (i *Index) Close() error { _ = testhook.Closed(i.holder.Auditor, i, nil) }() - err := i.Txf.CloseIndex(i) + err := i.holder.txf.CloseIndex(i) if err != nil { return errors.Wrap(err, "closing index") } @@ -424,8 +402,12 @@ func (i *Index) Close() error { return nil } +// make it clear what the Index.AvailableShards() calls are trying to obtain. +const includeRemote = false +const localOnly = true + // AvailableShards returns a bitmap of all shards with data in the index. -func (i *Index) AvailableShards() *roaring.Bitmap { +func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap { if i == nil { return roaring.NewBitmap() } @@ -435,8 +417,8 @@ func (i *Index) AvailableShards() *roaring.Bitmap { b := roaring.NewBitmap() for _, f := range i.fields { - //b.Union(f.AvailableShards()) - b.UnionInPlace(f.AvailableShards()) + //b.Union(f.AvailableShards(localOnly)) + b.UnionInPlace(f.AvailableShards(localOnly)) } i.Stats.Gauge(MetricMaxShard, float64(b.Max()), 1.0) @@ -445,7 +427,7 @@ func (i *Index) AvailableShards() *roaring.Bitmap { // Begin starts a transaction on a shard of the index. func (i *Index) BeginTx(writable bool, shard uint64) (Tx, error) { - return i.Txf.NewTx(Txo{Write: writable, Index: i, Shard: shard}), nil + return i.holder.txf.NewTx(Txo{Write: writable, Index: i, Shard: shard}), nil } // fieldPath returns the path to a field in the index. @@ -621,7 +603,7 @@ func (i *Index) DeleteField(name string) error { return errors.Wrap(err, "closing") } - if err := i.Txf.DeleteFieldFromStore(i.name, name, i.fieldPath(name)); err != nil { + if err := i.holder.txf.DeleteFieldFromStore(i.name, name, i.fieldPath(name)); err != nil { return errors.Wrap(err, "Txf.DeleteFieldFromStore") } @@ -705,7 +687,7 @@ func FormatQualifiedIndexName(index string) string { func (idx *Index) Dump(label string) { //fileline := FileLine(2) fmt.Printf("\nDump: %v\n\n", label) - idx.Txf.dbPerShard.DumpAll() + idx.holder.txf.dbPerShard.DumpAll() } func (idx *Index) SliceOfShards(field, view, viewPath string) (sliceOfShards []uint64, err error) { @@ -843,3 +825,7 @@ func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool) { fmt.Fprintf(w, "empty index '%v'", idx.path) } } + +func (idx *Index) Txf() *TxFactory { + return idx.holder.txf +} diff --git a/index_internal_test.go b/index_internal_test.go index bdcba3989..faed407c5 100644 --- a/index_internal_test.go +++ b/index_internal_test.go @@ -47,7 +47,7 @@ func (i *Index) reopen() error { if err := i.Close(); err != nil { return err } - if err := i.Open(false); err != nil { + if err := i.Open(); err != nil { return err } return nil diff --git a/license.exceptions b/license.exceptions index 1993f71b2..ffb08a26e 100644 --- a/license.exceptions +++ b/license.exceptions @@ -18,3 +18,4 @@ ./gid.go ./cmd/pilosa-keydump/vprint.go ./cmd/pilosa-keydump/keydump.go +./synthload/vprint.go diff --git a/lmdb.go b/lmdb.go index f652b6d7e..dcf2bde68 100644 --- a/lmdb.go +++ b/lmdb.go @@ -69,6 +69,17 @@ type lmdbRegistrar struct { path2db map[string]*LMDBWrapper } +func (r *lmdbRegistrar) Size() int { + r.mu.Lock() + defer r.mu.Unlock() + nmp := len(r.mp) + npa := len(r.path2db) + if nmp != npa { + panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa)) + } + return nmp +} + var globalLMDBReg *lmdbRegistrar = newLMDBTestRegistrar() var globalNextTxSnLMDB int64 @@ -101,18 +112,19 @@ func (r *lmdbRegistrar) unregister(w *LMDBWrapper) { } func DumpAllLMDB() { + short := true globalLMDBReg.mu.Lock() defer globalLMDBReg.mu.Unlock() for w := range globalLMDBReg.mp { - AlwaysPrintf("this lmdb path='%v' has: \n%v\n", w.path, w.StringifiedLMDBKeys(nil)) + AlwaysPrintf("this lmdb path='%v' has: \n%v\n", w.path, w.StringifiedLMDBKeys(nil, short)) } } // lmdbPath is a helper for determining the full directory // in which the lmdb database will be stored. func lmdbPath(path string) string { - if !strings.HasSuffix(path, "-lmdb") { - return path + "-lmdb" + if !strings.HasSuffix(path, "-lmdb@") { + return path + "-lmdb@" } return path } @@ -154,6 +166,9 @@ func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper //flags := uint(lmdb.NoReadahead | lmdb.NoSubdir) //flags := uint(lmdb.NoSubdir) // no difference without the No.Readahead on ./query. + // NoReadahead should be better for random workloads or those bigger than memory, + // as it avoids loading extra pages which then evict pages you are using. + //flags := uint(lmdb.NoReadahead) flags := uint(0) // unsafe, but get upper bound on performance. @@ -162,7 +177,15 @@ func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper // NoSync = C.MDB_NOSYNC // Don't fsync after commit. // flags = flags | lmdb.WriteMap | lmdb.NoMetaSync | lmdb.NoSync // about the same speed // flags = flags | lmdb.NoMetaSync | lmdb.NoSync // slows things down - //flags = flags | lmdb.WriteMap // seems faster than without: + + // TODO(jea): we got an odd segfault in the roaring/ pkg when we went to read-only mmap + // with the row-cache off; probably means we were trying to write to mmap-ed + // memory. Investigate at some point. reference: 2a77b25d ja/write_map_prevents_segfault + // + // Update: added COW to roaring/roaring.go:3282 func (c *Container) arrayRemove(), + // seems to fix the issue. + // + //flags = flags | lmdb.WriteMap // seems faster than without: or maybe not. not sure. // kRemove N= 710401 avg/op: 7.714µs sd: 27.83µs total: 5.480656859s // kAdd N= 722835 avg/op: 9.096µs sd: 105.787µs total: 6.575497725s @@ -225,6 +248,38 @@ func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper return w, nil } +func (w *LMDBWrapper) Path() string { + return w.path +} + +func (w *LMDBWrapper) HasData() (has bool, err error) { + + tx, err := w.NewTx(!writable, "", Txo{}) + if err != nil { + return false, errors.Wrap(err, "HasData NewTx") + } + defer tx.Rollback() + + bi := NewLMDBIterator(tx.(*LMDBTx), nil) + defer bi.Close() + + for bi.Next() { + return true, nil + } + return false, nil +} + +func (w *LMDBWrapper) CleanupTx(tx Tx) { + // inlined into Rollback and Commit, so this is a no-op, just here to satisfy the interface. +} + +func (tx *LMDBTx) IsDone() (done bool) { + tx.mu.Lock() + done = tx.unlocked + tx.mu.Unlock() + return +} + func (w *LMDBWrapper) OpenListString() (r string) { list := w.listopen() @@ -279,8 +334,6 @@ func (w *LMDBWrapper) DeleteIndex(indexName string) error { var _ Tx = (*LMDBTx)(nil) // LMDBWrapper provides the NewTx() method. -// Execute lmdbJob's via LMDBWrapper.submit(); these must -// be done by the lmdb goroutine worker pool. type LMDBWrapper struct { env *lmdb.Env @@ -310,6 +363,11 @@ type LMDBWrapper struct { openTx map[*LMDBTx]bool } +func (w *LMDBWrapper) SetHolder(h *Holder) { + // don't need it at the moment + //w.h = h +} + // NewTxWRITE lets us see in the callstack dumps where the WRITE tx are. // Can't have more than one active write per database, so the // 2nd one will block until the first finishes. @@ -321,6 +379,9 @@ func (w *LMDBWrapper) NewTxWRITE() *lmdb.Txn { // NewTxREAD lets us see in the callstack dumps where the READ tx are. func (w *LMDBWrapper) NewTxREAD() *lmdb.Txn { + if w.env == nil { + panic("cannot call NewTxREAD() on closed LMDBWrapper() -- open a new Wrapper first") + } lmdbTxn, err := w.env.BeginTxn(nil, lmdb.Readonly) panicOn(err) return lmdbTxn @@ -342,7 +403,8 @@ func (w *LMDBWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, sn := atomic.AddInt64(&globalNextTxSnLMDB, 1) - //vv("lmdb new tx _sn_ %v; stack \n%v", sn, stack()) + //vv("lmdb new tx _sn_ %v; openTx='%v', stack \n%v", sn, w.OpenListString(), stack()) + //vv("lmdb new (write=%v, shard=%v) tx _sn_ %v; openTx='%v'", write, o.Shard, sn, w.OpenListString()) runtime.LockOSThread() @@ -473,7 +535,7 @@ func (tx *LMDBTx) Type() string { } func (tx *LMDBTx) UseRowCache() bool { - return true + return false } // Pointer gives us a memory address for the underlying transaction for debugging. @@ -502,6 +564,7 @@ func (tx *LMDBTx) Rollback() { if !tx.unlocked { runtime.UnlockOSThread() tx.unlocked = true + tx.o.dbs.Cleanup(tx) } } @@ -527,6 +590,7 @@ func (tx *LMDBTx) Commit() error { if !tx.unlocked { runtime.UnlockOSThread() tx.unlocked = true + tx.o.dbs.Cleanup(tx) } return err } @@ -1244,6 +1308,7 @@ func (tx *LMDBTx) CountRange(index, field, view string, shard uint64, start, end // fragment.rowFromStorage() in fragment.go. For example: // // data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard, +// // f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) // ^ offset ^ start ^ endx // @@ -1252,6 +1317,10 @@ func (tx *LMDBTx) CountRange(index, field, view string, shard uint64, start, end // is done to conform to the roaring.OffsetRange() argument convention. // func (tx *LMDBTx) OffsetRange(index, field, view string, shard, offset, start, endx uint64) (other *roaring.Bitmap, err error) { + //vv("top of LMDBTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v)", index, field, view, int(shard), int(offset), int(start), int(endx)) + //defer func() { + //vv("returning from LMDBTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) other returning is: '%#v' stack=\n%v", index, field, view, int(shard), int(offset), int(start), int(endx), asInts(other.Slice()), stack()) + //}() // roaring does these three checks in its OffsetRange if lowbits(offset) != 0 { @@ -1469,11 +1538,11 @@ func ToContainer(typ byte, w []byte) (r *roaring.Container) { // StringifiedLMDBKeys returns a string with all the container // keys available in lmdb. -func (w *LMDBWrapper) StringifiedLMDBKeys(optionalUseThisTx Tx) (r string) { +func (w *LMDBWrapper) StringifiedLMDBKeys(optionalUseThisTx Tx, short bool) (r string) { if optionalUseThisTx == nil { tx, _ := w.NewTx(!writable, "", Txo{}) defer tx.Rollback() - r = stringifiedLMDBKeysTx(tx.(*LMDBTx)) + r = stringifiedLMDBKeysTx(tx.(*LMDBTx), short) return } @@ -1481,7 +1550,7 @@ func (w *LMDBWrapper) StringifiedLMDBKeys(optionalUseThisTx Tx) (r string) { if !ok { return fmt.Sprintf("", optionalUseThisTx) } - r = stringifiedLMDBKeysTx(btx) + r = stringifiedLMDBKeysTx(btx, short) return } @@ -1507,8 +1576,8 @@ func (tx *LMDBTx) countBitsSet(bkey []byte) (n int) { return } -func (tx *LMDBTx) Dump() { - fmt.Printf("%v\n", stringifiedLMDBKeysTx(tx)) +func (tx *LMDBTx) Dump(short bool) { + fmt.Printf("%v\n", stringifiedLMDBKeysTx(tx, short)) } // stringifiedLMDBKeysTx reports all the lmdb keys and a @@ -1520,7 +1589,7 @@ func (tx *LMDBTx) Dump() { // By convention, we must return the empty string if there // are no keys present. The tests use this to confirm // an empty database. -func stringifiedLMDBKeysTx(tx *LMDBTx) (r string) { +func stringifiedLMDBKeysTx(tx *LMDBTx, short bool) (r string) { r = "allkeys:[\n" it := NewLMDBIterator(tx, nil) @@ -1547,7 +1616,9 @@ func stringifiedLMDBKeysTx(tx *LMDBTx) (r string) { srbm = BitmapAsString(rbm) r += fmt.Sprintf("%v -> %v (%v hot)\n", key, h, tx.countBitsSet(bkey)) - r += " ......." + srbm + "\n" + if !short { + r += " ......." + srbm + "\n" + } } r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) diff --git a/lmdb_other.go b/lmdb_other.go index d19de8114..3a5c04995 100644 --- a/lmdb_other.go +++ b/lmdb_other.go @@ -63,6 +63,10 @@ func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrapper panic("lmdb only available on 64-bit arch") } +func (r *lmdbRegistrar) Size() int { + panic("lmdb only available on 64-bit arch") +} + // register each lmdb created under tests, so we // can clean them up. This is called by openLMDBWrapper() while // holding the r.mu.Lock, since it needs to atomically @@ -396,7 +400,11 @@ func (tx *LMDBTx) countBitsSet(bkey []byte) (n int) { panic("lmdb only available on 64-bit arch") } -func (tx *LMDBTx) Dump() { +func (tx *LMDBTx) IsDone() (done bool) { + panic("lmdb only available on 64-bit arch") +} + +func (tx *LMDBTx) Dump(short bool) { panic("lmdb only available on 64-bit arch") } diff --git a/lmdb_test.go b/lmdb_test.go index 8c4a885ae..2997b6c1e 100644 --- a/lmdb_test.go +++ b/lmdb_test.go @@ -94,7 +94,7 @@ func mustOpenEmptyLMDBWrapper(path string) (w *LMDBWrapper, cleaner func()) { w = ww.(*LMDBWrapper) // verify it is empty - allkeys := w.StringifiedLMDBKeys(nil) + allkeys := w.StringifiedLMDBKeys(nil, false) if allkeys != "" { panic(fmt.Sprintf("freshly created database was not empty! had keys:'%v'", allkeys)) } @@ -1013,7 +1013,7 @@ func TestLMDB_ImportRoaringBits(t *testing.T) { if n != 0 { panic(fmt.Sprintf("n = %v not zero so the clearbits didn't happen!", n)) } - allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx)) + allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx), false) // should have no keys if allkeys != "" { @@ -1185,7 +1185,7 @@ func TestLMDB_DeleteIndex(t *testing.T) { exists, err = tx.Contains(index, field, view, shard, v) panicOn(err) if exists { - allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx)) + allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx), false) panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) } } @@ -1240,7 +1240,7 @@ func TestLMDB_DeleteIndex_over100k(t *testing.T) { exists, err = tx.Contains(index, field, view, shard, v<<16) panicOn(err) if exists { - allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx)) + allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx), false) panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) } } @@ -1268,3 +1268,33 @@ func TestLMDB_SliceOfShards(t *testing.T) { } } } + +func TestLMDB_HasData(t *testing.T) { + + db, clean := mustOpenEmptyLMDBWrapper("TestLMDB_SliceOfShards") + defer clean() + defer db.Close() + + // HasData should start out false. + hasAnything, err := db.HasData() + if err != nil { + t.Fatal(err) + } + if hasAnything { + t.Fatalf("HasData reported existing data on an empty database") + } + + // check that HasData sees a committed record. + + index, field, view, shard, putme := "i", "f", "v", uint64(123), uint64(42) + LMDBMustSetBitvalue(db, index, field, view, shard, putme) + + // HasData(false) should now report data + hasAnything, err = db.HasData() + if err != nil { + t.Fatal(err) + } + if !hasAnything { + t.Fatalf("HasData() reported no data on a database that has bits written to it") + } +} diff --git a/mmap_test.go b/mmap_test.go index 9df7e60e6..52137d043 100644 --- a/mmap_test.go +++ b/mmap_test.go @@ -36,7 +36,7 @@ func forceSnapshotsCheckMapping(t *testing.T) { f.Logger = logger.NewLogfLogger(t) defer f.Clean(t) - tx = idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() for i := 0; i < f.MaxOpN; i++ { diff --git a/rbf.go b/rbf.go index 6317022e7..33fcd75c3 100644 --- a/rbf.go +++ b/rbf.go @@ -34,7 +34,7 @@ import ( // RbfDBWrapper wraps an *rbf.DB type RbfDBWrapper struct { - Path string + path string db *rbf.DB reg *rbfDBRegistrar muDb sync.Mutex @@ -49,6 +49,34 @@ type RbfDBWrapper struct { doAllocZero bool } +func (w *RbfDBWrapper) Path() string { + return w.path +} + +func (w *RbfDBWrapper) SetHolder(h *Holder) { + // don't need it at the moment + //w.h = h +} + +func (w *RbfDBWrapper) CleanupTx(tx Tx) { + r := tx.(*RBFTx) + r.mu.Lock() + if r.done { + r.mu.Unlock() + return + } + r.done = true + r.mu.Unlock() + + // try not to old r.mu while locking w.muDb + w.muDb.Lock() + + delete(w.openTx, r) + r.o.dbs.Cleanup(tx) // release the read/write lock. + + w.muDb.Unlock() +} + // rbfDBRegistrar also allows opening the same path twice to // result in sharing the same open database handle, and // thus the same transactional guarantees. @@ -60,6 +88,17 @@ type rbfDBRegistrar struct { path2db map[string]*RbfDBWrapper } +func (r *rbfDBRegistrar) Size() int { + r.mu.Lock() + defer r.mu.Unlock() + nmp := len(r.mp) + npa := len(r.path2db) + if nmp != npa { + panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa)) + } + return nmp +} + var globalRbfDBReg *rbfDBRegistrar = newRbfDBRegistrar() func newRbfDBRegistrar() *rbfDBRegistrar { @@ -77,29 +116,29 @@ func newRbfDBRegistrar() *rbfDBRegistrar { // return the existing instance. func (r *rbfDBRegistrar) unprotectedRegister(w *RbfDBWrapper) { r.mp[w] = true - r.path2db[w.Path] = w + r.path2db[w.path] = w } // unregister removes w from r func (r *rbfDBRegistrar) unregister(w *RbfDBWrapper) { r.mu.Lock() delete(r.mp, w) - delete(r.path2db, w.Path) + delete(r.path2db, w.path) r.mu.Unlock() } // rbfPath is a helper for determining the full directory // in which the RBF database will be stored. func rbfPath(path string) string { - if !strings.HasSuffix(path, "-rbfdb") { - return path + "-rbfdb" + if !strings.HasSuffix(path, "-rbfdb@") { + return path + "-rbfdb@" } return path } // OpenDBWrapper opens the database in the path directoy // without deleting any prior content. Any -// database directory will have the "-rbfdb" suffix. +// database directory will have the "-rbfdb@" suffix. // // OpenDBWrapper will check the registry and make a new instance only // if one does not exist for its path. Otherwise it returns @@ -114,18 +153,21 @@ func (r *rbfDBRegistrar) OpenDBWrapper(path0 string, doAllocZero bool) (DBWrappe // creates the effect of having only one DB open per pilosa node. return w, nil } - db := rbf.NewDB(path) + var db *rbf.DB + if doAllocZero { + db = rbf.NewDBWithAllocZero(path) + } else { + db = rbf.NewDB(path) + } w = &RbfDBWrapper{ reg: r, - Path: path, + path: path, db: db, doAllocZero: doAllocZero, openTx: make(map[*RBFTx]bool), } - r.unprotectedRegister(w) - rbf.DoAllocZero = doAllocZero err := db.Open() if err != nil { @@ -143,6 +185,16 @@ type RBFTx struct { o Txo sn int64 // serial number Db *RbfDBWrapper + + done bool + mu sync.Mutex // protect done as it changes state +} + +func (tx *RBFTx) IsDone() (done bool) { + tx.mu.Lock() + done = tx.done + tx.mu.Unlock() + return } func (tx *RBFTx) DBPath() string { @@ -154,19 +206,18 @@ func (tx *RBFTx) Type() string { } func (tx *RBFTx) Rollback() { - tx.Db.muDb.Lock() - delete(tx.Db.openTx, tx) - tx.Db.muDb.Unlock() - tx.tx.Rollback() + + // must happen after actual rollback + tx.Db.CleanupTx(tx) } -func (tx *RBFTx) Commit() error { - tx.Db.muDb.Lock() - delete(tx.Db.openTx, tx) - tx.Db.muDb.Unlock() +func (tx *RBFTx) Commit() (err error) { + err = tx.tx.Commit() - return tx.tx.Commit() + // must happen after actual commit + tx.Db.CleanupTx(tx) + return } func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { @@ -375,8 +426,8 @@ func (tx *RBFTx) Pointer() string { return fmt.Sprintf("%p", tx) } -func (tx *RBFTx) Dump() { - tx.tx.Dump() +func (tx *RBFTx) Dump(short bool) { + tx.tx.Dump(short) } // Readonly is true if the transaction is not read-and-write, but only doing reads. @@ -416,6 +467,12 @@ func rbfFieldPrefix(index, field string) string { return string(txkey.FieldPrefix(index, field)) } +func (w *RbfDBWrapper) HasData() (has bool, err error) { + w.muDb.Lock() + defer w.muDb.Unlock() + return w.db.HasData(false) // false => any prior attempt at write means we "have data" +} + func (w *RbfDBWrapper) DeleteField(index, field, fieldPath string) error { w.muDb.Lock() defer w.muDb.Unlock() diff --git a/rbf/cursorx.go b/rbf/cursorx.go index bca2c4491..7d2d9332c 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -26,10 +26,13 @@ import ( // if enableRowCache, then we must not return mmap-ed memory // directly, but only a copy. -const EnableRowCache = true +const EnableRowCache = false -// makes a copy, BUT doesn't do the zero out for now TODO(jea) zero out actually to detect -// access past tx. +// DoAllocZero means we copy mmap read data and +// wipe it afterwards to catch retention of data +// past Tx.Rollback which was a big problem. +// This should be set by NewDBWithAllocZero and never changed +// afterwards in order to avoid a data race. var DoAllocZero bool //probably should just implement the container interface diff --git a/rbf/db.go b/rbf/db.go index c8b4cf9ac..2554dfc08 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -58,6 +58,14 @@ type DB struct { MaxSize int64 } +// NewDBWithAllocZero sets DoAllocZero true and +// returns a new instance of DB. We set it here +// to avoid a data race afterwards. +func NewDBWithAllocZero(path string) *DB { + DoAllocZero = true + return NewDB(path) +} + // NewDB returns a new instance of DB. func NewDB(path string) *DB { db := &DB{ @@ -544,6 +552,68 @@ func (db *DB) closeWALSegments() (err error) { return err } +// HasData with requireOneHotBit=false returns +// hasAnyRecords true if any record has been stored, +// even if the value for that bitmap record turned out to have +// no bits hot (be all zeroes). +// +// In this case, we are taking the attempted storage +// of any named bitmap into the database as evidence +// that the db is in use, and we return hasAnyRecords true. +// +// Conversely, if requireOneHotBit is true, then a +// database consisting of only a named bitmap with +// an all zeroes (no bits hot) +// will return hasAnyRecords false. We must find at +// least a single hot bit inside the db +// in order to return hasAnyRecords true. +// +// HasData is used by backend migration and blue/green checks. +// +// If there is a disk error we return (false, error), so always +// check the error before deciding if hasAnyRecords is valid. +// +// We will internally create and rollback a read-only +// transaction to answer this query. +func (db *DB) HasData(requireOneHotBit bool) (hasAnyRecords bool, err error) { + + // Read a list of all bitmaps in Tx. + tx, err := db.Begin(false) + if err != nil { + return false, err + } + defer tx.Rollback() + + records, err := tx.RootRecords() + if err != nil { + return false, err + } + // Loop over each bitmap and attempt to move to the first cell. + // If we can move to a cell then we have at least one record. + for _, record := range records { + // Fetch cursor for bitmap. + cur, err := tx.Cursor(record.Name) + if err != nil { + return false, err + } else if cur == nil { + continue // no bitmap + } + if !requireOneHotBit { + return true, nil + } + // INVAR: requireOneHotBit true + + // Check if we can move to the first cell. + if err := cur.First(); err == io.EOF { + continue // no data in bitmap + } else if err != nil { + return false, err + } + return true, nil + } + return false, nil +} + // Size returns the size of the database & WAL, in bytes. func (db *DB) Size() (int64, error) { diff --git a/rbf/db_test.go b/rbf/db_test.go index 667fbf54d..98a70cb24 100644 --- a/rbf/db_test.go +++ b/rbf/db_test.go @@ -191,3 +191,83 @@ func TestDB_BeginWithExclusiveLock(t *testing.T) { } }) } + +func TestDB_HasData(t *testing.T) { + + db := MustOpenDB(t) + defer MustCloseDB(t, db) + + // HasData should start out false. + const requireOneHotBit = true + hasAnything, err := db.HasData(!requireOneHotBit) + if err != nil { + t.Fatal(err) + } + if hasAnything { + t.Fatalf("HasData reported existing data on an empty database") + } + + hasAnything, err = db.HasData(requireOneHotBit) + if err != nil { + t.Fatal(err) + } + if hasAnything { + t.Fatalf("HasData reported existing data on an empty database") + } + + // check that HasData sees a committed record. + + // Create bitmap with no hot bits. + if tx, err := db.Begin(true); err != nil { + t.Fatal(err) + } else if err := tx.CreateBitmap("x"); err != nil { + t.Fatal(err) + } else if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + // HasData(false) should now report seeing the 'x' record, even though + // the value is an empty bitmap, since !requireOneHotBit. + hasAnything, err = db.HasData(!requireOneHotBit) + if err != nil { + t.Fatal(err) + } + if !hasAnything { + t.Fatalf("HasData(!requireOneHotBit) reported no data on a database that has 'x' written to it") + } + + // HasData(requireOneHotBit) should report false, since we have no hot bits. + hasAnything, err = db.HasData(requireOneHotBit) + if err != nil { + t.Fatal(err) + } + if hasAnything { + t.Fatalf("HasData(requireOneHotBit) reported data on a database that has 'x' -> empty bitmap") + } + + // hot up a bit. + if tx, err := db.Begin(true); err != nil { + t.Fatal(err) + } else if _, err := tx.Add("x", rand.Uint64()); err != nil { + t.Fatal(err) + } else if err := tx.Commit(); err != nil { + t.Fatal(err) + } + + // Now it shouldn't matter, we should get HasData true either way + // HasData(requireOneHotBit) should report false, since we have no hot bits. + hasAnything, err = db.HasData(requireOneHotBit) + if err != nil { + t.Fatal(err) + } + if !hasAnything { + t.Fatalf("HasData should have seen the hot bit") + } + hasAnything, err = db.HasData(!requireOneHotBit) + if err != nil { + t.Fatal(err) + } + if !hasAnything { + t.Fatalf("HasData should have seen the hot bit") + } +} diff --git a/rbf/tx.go b/rbf/tx.go index d1d4f1ad4..bf1093c7e 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -1334,10 +1334,10 @@ func (si *emptyContainerIterator) Value() (uint64, *roaring.Container) { panic("emptyContainerIterator never has any Values") } -func (tx *Tx) Dump() { - fmt.Println(tx.DumpString()) +func (tx *Tx) Dump(short bool) { + fmt.Println(tx.DumpString(short)) } -func (tx *Tx) DumpString() (r string) { +func (tx *Tx) DumpString(short bool) (r string) { r = "allkeys:[\n" @@ -1365,7 +1365,7 @@ func (tx *Tx) DumpString() (r string) { ckey := cell.Key ct := toContainer(cell, tx) - s := stringOfCkeyCt(ckey, ct, rr.Name) + s := stringOfCkeyCt(ckey, ct, rr.Name, short) r += s n++ } @@ -1419,7 +1419,7 @@ func bitmapAsString(rbm *roaring.Bitmap) (r string) { return r + ")" } -func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string) (s string) { +func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string, short bool) (s string) { by := containerToBytes(ct) hash := hash.Blake3sum16(by) @@ -1433,7 +1433,9 @@ func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string) (s string bkey := pre + fmt.Sprintf("ckey@%020d", ckey) s = fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, ct.N()) - s += " ......." + srbm + "\n" + if !short { + s += " ......." + srbm + "\n" + } return } diff --git a/rbf/tx_test.go b/rbf/tx_test.go index b9bc99373..fedd5f47d 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -483,7 +483,7 @@ func TestTx_Dump(t *testing.T) { } // test that we don't crash, and get *something* back - s := tx.DumpString() + s := tx.DumpString(true) if s == "" { panic("should have had 3 containers!") } diff --git a/roaring/fuzzer.go b/roaring/fuzzer.go index 9312f7865..e76ea397e 100644 --- a/roaring/fuzzer.go +++ b/roaring/fuzzer.go @@ -191,14 +191,14 @@ func FuzzRoaringOps(data []byte) int { expected = make([]uint64, 0) actual = make([]uint64, 0) forEachInSlice(s1, func(v uint64) { expected = append(expected, v) }) - bm1.ForEach(func(v uint64) { actual = append(actual, v) }) + bm1.ForEach(func(v uint64) error { actual = append(actual, v); return nil }) if !reflect.DeepEqual(expected, actual) { panic(fmt.Sprintf("for each:\n expected %v\n got %v", expected, actual)) } expected = make([]uint64, 0) actual = make([]uint64, 0) forEachInSlice(s2, func(v uint64) { expected = append(expected, v) }) - bm2.ForEach(func(v uint64) { actual = append(actual, v) }) + bm2.ForEach(func(v uint64) error { actual = append(actual, v); return nil }) if !reflect.DeepEqual(expected, actual) { panic(fmt.Sprintf("for each:\n expected %v\n got %v", expected, actual)) } @@ -206,14 +206,14 @@ func FuzzRoaringOps(data []byte) int { expected = make([]uint64, 0) actual = make([]uint64, 0) forEachInRangeSlice(s1, start, end, func(v uint64) { expected = append(expected, v) }) - bm1.ForEachRange(start, end, func(v uint64) { actual = append(actual, v) }) + bm1.ForEachRange(start, end, func(v uint64) error { actual = append(actual, v); return nil }) if !reflect.DeepEqual(expected, actual) { panic(fmt.Sprintf("for each in range:\n expected %v\n got %v", expected, actual)) } expected = make([]uint64, 0) actual = make([]uint64, 0) forEachInRangeSlice(s2, start, end, func(v uint64) { expected = append(expected, v) }) - bm2.ForEachRange(start, end, func(v uint64) { actual = append(actual, v) }) + bm2.ForEachRange(start, end, func(v uint64) error { actual = append(actual, v); return nil }) if !reflect.DeepEqual(expected, actual) { panic(fmt.Sprintf("for each in range:\n expected %v\n got %v", expected, actual)) } diff --git a/roaring/roaring.go b/roaring/roaring.go index eabec3b86..52e806f52 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3301,8 +3301,32 @@ func (c *Container) arrayRemove(v uint16) (*Container, bool) { c = c.Thaw() array = c.array() - array = append(array[:i], array[i+1:]...) - c.setArray(array) + const needCopyOnWriteDueToReadOnlyMmap = true + + // TODO(jea) quick benchmarks don't show less performance if needCopyOnWriteDueToReadOnlyMmap + // is true, but we may need more rigorous measurement. + // + // Don't have a COW: + // all benchmarks in roaring/ + // ok github.com/pilosa/pilosa/v2/roaring 217.138s + // + // ok, have a COW: + // all benchmarks in roaring/ + // ok github.com/pilosa/pilosa/v2/roaring 214.295s + + if needCopyOnWriteDueToReadOnlyMmap { + n := len(array) + array2 := make([]uint16, n-1) + copy(array2, array[:i]) + copy(array2[i:], array[i+1:]) + c.setArray(array2) + } else { + // seg fault here with read-only mmap; go 1.14.7 linux. + // example: cap = 7 i = 0 len = 7 + // the append tries to write read-only memory? + array = append(array[:i], array[i+1:]...) + c.setArray(array) + } return c, true } diff --git a/rrtx.go b/rrtx.go index 9ef266c60..597a61ffa 100644 --- a/rrtx.go +++ b/rrtx.go @@ -21,6 +21,7 @@ import ( "os" "path/filepath" "strconv" + "sync" "sync/atomic" "github.com/pilosa/pilosa/v2/roaring" @@ -35,18 +36,30 @@ type RoaringTx struct { fragment *fragment o Txo sn int64 // serial number + + done bool + mu sync.Mutex // protect done as it changes state + + w *RoaringWrapper +} + +func (tx *RoaringTx) IsDone() (done bool) { + tx.mu.Lock() + done = tx.done + tx.mu.Unlock() + return } func (tx *RoaringTx) Type() string { return RoaringTxn } -func (tx *RoaringTx) Dump() { - fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys(false, false)) +func (tx *RoaringTx) Dump(short bool) { + fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys(short, false, tx.o)) } func (tx *RoaringTx) UseRowCache() bool { - return true + return false } func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { @@ -125,11 +138,14 @@ func (tx *RoaringTx) IncrementOpN(index, field, view string, shard uint64, chang frag.incrementOpN(changedN) } -// Rollback is a no-op as Roaring does not support transactions. -func (tx *RoaringTx) Rollback() {} +// Rollback +func (tx *RoaringTx) Rollback() { + tx.w.CleanupTx(tx) +} -// Commit is a no-op as Roaring does not support transactions. +// Commit func (tx *RoaringTx) Commit() error { + tx.w.CleanupTx(tx) return nil } @@ -359,80 +375,6 @@ func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.B return frag.storage, nil } -var globalRoaringReg = &RoaringStore{} - -func (r *RoaringStore) OpenDBWrapper(path string, doAllocZero bool) (DBWrapper, error) { - return r, nil -} - -func (w *RoaringStore) DeleteDBPath(dbs *DBShard) error { - return os.RemoveAll(dbs.Path) -} - -func (r *RoaringStore) Close() error { - return nil -} - -func (w *RoaringStore) OpenListString() (r string) { - return "RoaringStore.OpenListString() not yet implemented" -} - -func (w *RoaringStore) OpenSnList() (sns []int64) { - return nil -} - -type RoaringStore struct{} - -func NewRoaringStore() *RoaringStore { - return &RoaringStore{} -} - -var globalNextTxSnRoaring int64 - -func (db *RoaringStore) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { - sn := atomic.AddInt64(&globalNextTxSnRoaring, 1) - return &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment, o: o, sn: sn}, nil -} - -func (db *RoaringStore) DeleteField(index, field, fieldPath string) error { - - // match txn sn count vs lmdb/etc. - atomic.AddInt64(&globalNextTxSnRoaring, 1) - - // under blue-green badger_roaring, the directory will not be found, b/c badger will have - // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: - // "If the path does not exist, RemoveAll returns nil (no error)" - err := os.RemoveAll(fieldPath) - if err != nil { - return errors.Wrap(err, "removing directory") - } - return nil -} - -// frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil. -// The fragment should be closed before this. -func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { - - // match txn sn count vs lmdb/etc. - atomic.AddInt64(&globalNextTxSnRoaring, 1) - - fragment, ok := frag.(*fragment) - if !ok { - return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag) - } - - // Delete fragment file. - if err := os.Remove(fragment.path); err != nil { - return errors.Wrap(err, "deleting fragment file") - } - - // Delete fragment cache file. - if err := os.Remove(fragment.cachePath()); err != nil { - return errors.Wrap(err, fmt.Sprintf("no cache file to delete for shard %d", fragment.shard)) - } - return nil -} - func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { file, err := os.Open(fragmentPathForRoaring) // open the fragment file if err != nil { @@ -459,3 +401,193 @@ func (tx *RoaringTx) Options() Txo { func (tx *RoaringTx) Sn() int64 { return tx.sn } + +//////// registrar and wrapper machinery + +// roaringRegistrar mirrors the machinery expected +// for all backends for the roaring files approach. +// +type roaringRegistrar struct { + mu sync.Mutex + mp map[*RoaringWrapper]bool + + path2db map[string]*RoaringWrapper +} + +func (r *roaringRegistrar) Size() int { + r.mu.Lock() + defer r.mu.Unlock() + nmp := len(r.mp) + npa := len(r.path2db) + if nmp != npa { + panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa)) + } + return nmp +} + +var globalRoaringReg *roaringRegistrar = newRoaringRegistrar() + +func newRoaringRegistrar() *roaringRegistrar { + return &roaringRegistrar{ + mp: make(map[*RoaringWrapper]bool), + path2db: make(map[string]*RoaringWrapper), + } +} + +func (r *roaringRegistrar) unprotectedRegister(w *RoaringWrapper) { + r.mp[w] = true + r.path2db[w.path] = w +} + +// unregister removes w from r +func (r *roaringRegistrar) unregister(w *RoaringWrapper) { + r.mu.Lock() + delete(r.mp, w) + delete(r.path2db, w.path) + r.mu.Unlock() +} + +// openRoaringDB will check the registry and make a new instance only +// if one does not exist for its path0. Otherwise it returns +// the existing instance. +func (r *roaringRegistrar) OpenDBWrapper(path string, doAllocZero bool) (DBWrapper, error) { + + r.mu.Lock() + defer r.mu.Unlock() + w, ok := r.path2db[path] + if ok { + return w, nil + } + // otherwise, make a new roaring and store it in globalRoaringReg + w = &RoaringWrapper{ + reg: r, + path: path, + } + r.unprotectedRegister(w) + + return w, nil +} + +func (w *RoaringWrapper) SetHolder(h *Holder) { + w.h = h +} + +func (w *RoaringWrapper) Path() string { + return w.path +} + +func (w *RoaringWrapper) HasData() (has bool, err error) { + return w.h.HasRoaringData() +} + +func (w *RoaringWrapper) CleanupTx(tx Tx) { + r := tx.(*RoaringTx) + r.mu.Lock() + defer r.mu.Unlock() + if r.done { + return + } + r.done = true + + r.o.dbs.Cleanup(tx) // release the read/write lock. +} + +func (w *RoaringWrapper) OpenListString() (r string) { + return "RoaringWrapper.OpenListString() not yet implemented" +} + +func (w *RoaringWrapper) OpenSnList() (slc []int64) { + return nil +} + +// statically confirm that RoaringTx satisfies the Tx interface. +var _ Tx = (*RoaringTx)(nil) + +// RoaringWrapper provides the NewTx() method. +type RoaringWrapper struct { + muDb sync.Mutex + + path string + + h *Holder + + reg *roaringRegistrar + + // make RoaringWrapper.Close() idempotent, avoiding panic on double Close() + closed bool +} + +var globalNextTxSnRoaring int64 + +func (w *RoaringWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { + + sn := atomic.AddInt64(&globalNextTxSnRoaring, 1) + return &RoaringTx{ + write: o.Write, + Field: o.Field, + Index: o.Index, + fragment: o.Fragment, + o: o, + sn: sn, + w: w, + }, nil +} + +// Close shuts down the Roaring database. +func (w *RoaringWrapper) Close() (err error) { + w.muDb.Lock() + defer w.muDb.Unlock() + if !w.closed { + w.reg.unregister(w) + w.closed = true + } + return nil +} + +func (w *RoaringWrapper) IsClosed() (closed bool) { + w.muDb.Lock() + closed = w.closed + w.muDb.Unlock() + return +} + +func (w *RoaringWrapper) DeleteDBPath(dbs *DBShard) (err error) { + return os.RemoveAll(dbs.Path) +} + +func (w *RoaringWrapper) DeleteField(index, field, fieldPath string) error { + + // match txn sn count vs lmdb/etc. + atomic.AddInt64(&globalNextTxSnRoaring, 1) + + // under blue-green badger_roaring, the directory will not be found, b/c badger will have + // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: + // "If the path does not exist, RemoveAll returns nil (no error)" + err := os.RemoveAll(fieldPath) + if err != nil { + return errors.Wrap(err, "removing directory") + } + return nil +} + +func (w *RoaringWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { + + // match txn sn count vs lmdb/etc. + atomic.AddInt64(&globalNextTxSnRoaring, 1) + + fragment, ok := frag.(*fragment) + if !ok { + return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag) + } + + // Delete fragment file. + if err := os.Remove(fragment.path); err != nil { + return errors.Wrap(err, "deleting fragment file") + } + + // Delete fragment cache file. + if err := os.Remove(fragment.cachePath()); err != nil { + return errors.Wrap(err, fmt.Sprintf("no cache file to delete for shard %d", fragment.shard)) + } + return nil +} diff --git a/rrtx_internal_test.go b/rrtx_internal_test.go new file mode 100644 index 000000000..3a03ca763 --- /dev/null +++ b/rrtx_internal_test.go @@ -0,0 +1,63 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pilosa + +import ( + "os" + "testing" +) + +func TestRoaring_HasData(t *testing.T) { + + orig := os.Getenv("PILOSA_TXSRC") + defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests! + os.Setenv("PILOSA_TXSRC", "roaring") + + idx := newIndexWithTempPath(t, "i") + defer idx.Close() + + db, err := globalRoaringReg.OpenDBWrapper(idx.path, false) + panicOn(err) + db.SetHolder(idx.holder) + + // HasData should start out false. + hasAnything, err := db.HasData() + panicOn(err) + + if hasAnything { + t.Fatalf("HasData reported existing data on an empty database") + } + + // check that HasData sees a committed record. + + field, shard := "f", uint64(123) + + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) + defer tx.Rollback() + + f, err := idx.CreateField(field) + panicOn(err) + _, err = f.SetBit(tx, 1, 1, nil) + panicOn(err) + panicOn(tx.Commit()) + + hasAnything, err = db.HasData() + if err != nil { + t.Fatal(err) + } + if !hasAnything { + t.Fatalf("HasData() reported no data on a database that has 'x' written to it") + } +} diff --git a/server/server.go b/server/server.go index bfd54cead..1b7dea316 100644 --- a/server/server.go +++ b/server/server.go @@ -290,6 +290,9 @@ func (m *Command) SetupServer() error { txty := pilosa.MustTxsrcToTxtype(m.Config.Txsrc) // will panic on unknown Txsrc. os.Setenv("PILOSA_TXSRC", m.Config.Txsrc) m.logger.Printf("using Txsrc '%v'/%v", m.Config.Txsrc, txty) + if len(txty) == 2 { + m.logger.Printf("blue='%v' / green='%v'", txty[0], txty[1]) + } // validateAddrs sets the appropriate values for Bind and Advertise // based on the inputs. It is not responsible for applying defaults, although diff --git a/stattx.go b/stattx.go index 0309faf46..20b30a57c 100644 --- a/stattx.go +++ b/stattx.go @@ -310,8 +310,8 @@ func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) } -func (c *statTx) Dump() { - c.b.Dump() +func (c *statTx) Dump(short bool) { + c.b.Dump(short) } func (c *statTx) Readonly() bool { @@ -433,6 +433,10 @@ func (c *statTx) UseRowCache() bool { return c.b.UseRowCache() } +func (c *statTx) IsDone() (done bool) { + return c.b.IsDone() +} + func (c *statTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) { me := kAdd diff --git a/synthload/synthload_test.go b/synthload/synthload_test.go new file mode 100644 index 000000000..ec4c6b1ce --- /dev/null +++ b/synthload/synthload_test.go @@ -0,0 +1,361 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package synthload + +/* // still work in progress, comment out for now, since we + // are getting a hang on go1.13 in CI +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/server" + "github.com/pilosa/pilosa/v2/test" +) + +func Test_SynthLoad_ImportSchema(t *testing.T) { + c := test.MustRunCluster(t, 1, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID("node0"), + )}, + ) + defer c.Close() + + m0 := c.GetNode(0) + + tarball := "testindex.tar.gz" + + // get the holder.Path to write to + h := m0.API.Holder() + target := h.Path() + panicOn(h.Close()) + + panicOn(unpackTarball(tarball, target)) + + // reopen + panicOn(h.Open()) + + qs := strings.Split(pql, "\n\n") + //vv("qs = '%#v'", qs) + for _, q := range qs { + req := &pilosa.QueryRequest{ + // Index to execute query against. + Index: "testindex", + + // The query string to parse and execute. + Query: q, + + // // The shards to include in the query execution. + // // If empty, all shards are included. + // Shards []uint64 + + // // Return column attributes, if true. + // ColumnAttrs bool + + // // Do not return row attributes, if true. + // ExcludeRowAttrs bool + + // // Do not return columns, if true. + // ExcludeColumns bool + + // // If true, indicates that query is part of a larger distributed query. + // // If false, this request is on the originating node. + // Remote bool + + // // Should we profile this query? + // Profile bool + + // // Additional data associated with the query, in cases where there's + // // row-style inputs for precomputed values. + // EmbeddedData []*Row + } + + qr, err := m0.API.Query(context.Background(), req) + panicOn(err) + vv("qr = '%#v'", qr) + } +} + +var _ = applySchema + +func applySchema(m0 *test.Command, schemaStr string) { + // don't need schema now that we import the tarball, it has it all. + schema := &pilosa.Schema{} + err := json.NewDecoder(bytes.NewBufferString(schemaStr)).Decode(schema) + panicOn(err) + + ctx := context.Background() + remote := false + err = m0.API.ApplySchema(ctx, schema, remote) + panicOn(err) +} + +func unpackTarball(tarball, target string) error { + + vv("target = '%v'", target) + fd, err := os.Open(tarball) + panicOn(err) + defer fd.Close() + gz, err := gzip.NewReader(fd) + panicOn(err) + defer gz.Close() + + tarReader := tar.NewReader(gz) + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } else if err != nil { + panic(err) + } + + path := filepath.Join(target, header.Name) + info := header.FileInfo() + if info.IsDir() { + if err = os.MkdirAll(path, info.Mode()); err != nil { + panicOn(err) + } + continue + } + + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) + panicOn(err) + + _, err = io.Copy(file, tarReader) + panicOn(err) + file.Close() + } + return nil +} + +// mkdir o; bangbang schemator -o o +var pql = ` +Count(All()) + +Rows(field='bools',limit=10) +Count(Row(bools='bool')) +Count(Not(Row(bools='bool'))) +Count(Intersect(Row(bools='bool'))) +Count(Difference(Union(Row(bools='bool')),Intersect(Row(bools='bool')))) +GroupBy(Rows(field='bools'),limit=10) + +Rows(field='bools-exists',limit=10) +Count(Row(bools-exists='bool')) +Count(Not(Row(bools-exists='bool'))) +Count(Intersect(Row(bools-exists='bool'))) +Count(Difference(Union(Row(bools-exists='bool')),Intersect(Row(bools-exists='bool')))) +GroupBy(Rows(field='bools-exists'),limit=10) + +GroupBy(Rows(field='int'),limit=10) +Min(field='int') +Max(field='int') +Sum(field='int') + +Rows(field='mutex',limit=10) +Count(Row(mutex='12')) +Count(Not(Row(mutex='12'))) +Count(Row(mutex='13')) +Count(Not(Row(mutex='13'))) +Count(Row(mutex='0')) +Count(Not(Row(mutex='0'))) +Count(Row(mutex='5')) +Count(Not(Row(mutex='5'))) +Count(Row(mutex='2')) +Count(Not(Row(mutex='2'))) +Count(Row(mutex='14')) +Count(Not(Row(mutex='14'))) +Count(Row(mutex='3')) +Count(Not(Row(mutex='3'))) +Count(Row(mutex='18')) +Count(Not(Row(mutex='18'))) +Count(Row(mutex='7')) +Count(Not(Row(mutex='7'))) +Count(Row(mutex='19')) +Count(Not(Row(mutex='19'))) +Count(Intersect(Row(mutex='12'),Row(mutex='13'),Row(mutex='0'),Row(mutex='5'),Row(mutex='2'),Row(mutex='14'),Row(mutex='3'),Row(mutex='18'),Row(mutex='7'),Row(mutex='19'))) +Count(Difference(Union(Row(mutex='12'),Row(mutex='13'),Row(mutex='0'),Row(mutex='5'),Row(mutex='2'),Row(mutex='14'),Row(mutex='3'),Row(mutex='18'),Row(mutex='7'),Row(mutex='19')),Intersect(Row(mutex='12'),Row(mutex='13'),Row(mutex='0'),Row(mutex='5'),Row(mutex='2'),Row(mutex='14'),Row(mutex='3'),Row(mutex='18'),Row(mutex='7'),Row(mutex='19')))) +GroupBy(Rows(field='mutex'),limit=10) + +Rows(field='set',limit=10) +Count(Row(set='v621')) +Count(Not(Row(set='v621'))) +Count(Row(set='v997')) +Count(Not(Row(set='v997'))) +Count(Row(set='v772')) +Count(Not(Row(set='v772'))) +Count(Row(set='v340')) +Count(Not(Row(set='v340'))) +Count(Row(set='v766')) +Count(Not(Row(set='v766'))) +Count(Row(set='v416')) +Count(Not(Row(set='v416'))) +Count(Row(set='v481')) +Count(Not(Row(set='v481'))) +Count(Row(set='v581')) +Count(Not(Row(set='v581'))) +Count(Row(set='v591')) +Count(Not(Row(set='v591'))) +Count(Row(set='v675')) +Count(Not(Row(set='v675'))) +Count(Intersect(Row(set='v621'),Row(set='v997'),Row(set='v772'),Row(set='v340'),Row(set='v766'),Row(set='v416'),Row(set='v481'),Row(set='v581'),Row(set='v591'),Row(set='v675'))) +Count(Difference(Union(Row(set='v621'),Row(set='v997'),Row(set='v772'),Row(set='v340'),Row(set='v766'),Row(set='v416'),Row(set='v481'),Row(set='v581'),Row(set='v591'),Row(set='v675')),Intersect(Row(set='v621'),Row(set='v997'),Row(set='v772'),Row(set='v340'),Row(set='v766'),Row(set='v416'),Row(set='v481'),Row(set='v581'),Row(set='v591'),Row(set='v675')))) +GroupBy(Rows(field='set'),limit=10) + +Rows(field='string',limit=10) +Count(Row(string='KPFGWOYUGTCF')) +Count(Not(Row(string='KPFGWOYUGTCF'))) +Count(Row(string='VNSHWDTNELDA')) +Count(Not(Row(string='VNSHWDTNELDA'))) +Count(Row(string='LTDOGKKFGZPW')) +Count(Not(Row(string='LTDOGKKFGZPW'))) +Count(Row(string='EETNIIDCDZHB')) +Count(Not(Row(string='EETNIIDCDZHB'))) +Count(Row(string='ATWOMTRASGHP')) +Count(Not(Row(string='ATWOMTRASGHP'))) +Count(Row(string='KOJUBJQVMXZL')) +Count(Not(Row(string='KOJUBJQVMXZL'))) +Count(Row(string='NOXDEYPTGZBH')) +Count(Not(Row(string='NOXDEYPTGZBH'))) +Count(Row(string='DLSRPRADBPKX')) +Count(Not(Row(string='DLSRPRADBPKX'))) +Count(Row(string='WJUEQABAEEUX')) +Count(Not(Row(string='WJUEQABAEEUX'))) +Count(Row(string='PFRGBFDLHHPK')) +Count(Not(Row(string='PFRGBFDLHHPK'))) +Count(Intersect(Row(string='KPFGWOYUGTCF'),Row(string='VNSHWDTNELDA'),Row(string='LTDOGKKFGZPW'),Row(string='EETNIIDCDZHB'),Row(string='ATWOMTRASGHP'),Row(string='KOJUBJQVMXZL'),Row(string='NOXDEYPTGZBH'),Row(string='DLSRPRADBPKX'),Row(string='WJUEQABAEEUX'),Row(string='PFRGBFDLHHPK'))) +Count(Difference(Union(Row(string='KPFGWOYUGTCF'),Row(string='VNSHWDTNELDA'),Row(string='LTDOGKKFGZPW'),Row(string='EETNIIDCDZHB'),Row(string='ATWOMTRASGHP'),Row(string='KOJUBJQVMXZL'),Row(string='NOXDEYPTGZBH'),Row(string='DLSRPRADBPKX'),Row(string='WJUEQABAEEUX'),Row(string='PFRGBFDLHHPK')),Intersect(Row(string='KPFGWOYUGTCF'),Row(string='VNSHWDTNELDA'),Row(string='LTDOGKKFGZPW'),Row(string='EETNIIDCDZHB'),Row(string='ATWOMTRASGHP'),Row(string='KOJUBJQVMXZL'),Row(string='NOXDEYPTGZBH'),Row(string='DLSRPRADBPKX'),Row(string='WJUEQABAEEUX'),Row(string='PFRGBFDLHHPK')))) +GroupBy(Rows(field='string'),limit=10) + +GroupBy(Rows(field='time'),limit=10) +` + +// datagen --source=kitchensink --pilosa.index=testindex --start-from=1 --end-at=1000 --pilosa.batch-size=10000 --pilosa.hosts localhost:10101 +// curl localhost:10101/schema + +var _ = schemaString +var schemaString = ` +{ + "indexes": [ + { + "name": "testindex", + "createdAt": 1599601704641744600, + "options": { + "keys": false, + "trackExistence": true + }, + "fields": [ + { + "name": "bools", + "createdAt": 1599601704647053000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "bools-exists", + "createdAt": 1599601704643225900, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "decimal", + "createdAt": 1599601704642064400, + "options": { + "type": "decimal", + "base": 0, + "scale": 2, + "bitDepth": 63, + "min": -92233720368547760, + "max": 92233720368547760, + "keys": false + } + }, + { + "name": "int", + "createdAt": 1599601704645510100, + "options": { + "type": "int", + "base": 0, + "bitDepth": 63, + "min": -9223372036854776000, + "max": 9223372036854776000, + "keys": false, + "foreignIndex": "" + } + }, + { + "name": "mutex", + "createdAt": 1599601704647187200, + "options": { + "type": "mutex", + "cacheType": "ranked", + "cacheSize": 500, + "keys": true + } + }, + { + "name": "set", + "createdAt": 1599601704643664400, + "options": { + "type": "set", + "cacheType": "lru", + "cacheSize": 1, + "keys": true + } + }, + { + "name": "string", + "createdAt": 1599601704644342000, + "options": { + "type": "set", + "cacheType": "ranked", + "cacheSize": 50000, + "keys": true + } + }, + { + "name": "time", + "createdAt": 1599601704643013400, + "options": { + "type": "time", + "timeQuantum": "YMD", + "keys": true, + "noStandardView": false + } + } + ], + "shardWidth": 1048576 + } + ] +} +` + +*/ diff --git a/synthload/testindex.tar.gz b/synthload/testindex.tar.gz new file mode 100644 index 000000000..15cdfe2b6 Binary files /dev/null and b/synthload/testindex.tar.gz differ diff --git a/synthload/vprint.go b/synthload/vprint.go new file mode 100644 index 000000000..d9f8a03ec --- /dev/null +++ b/synthload/vprint.go @@ -0,0 +1,170 @@ +// home: https://github.com/glycerine/vprint +// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved. +// License: MIT +// +// MIT License +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package synthload + +import ( + "fmt" + "io" + "os" + "path" + "runtime" + "runtime/debug" + "sync" + "time" +) + +const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00" +const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00" + +// for tons of debug output +var VerboseVerbose bool = false + +// convience functions for . import +var pp = PP +var vv = VV + +var panicOn = PanicOn +var _ = panicOn + +func init() { + // keeper linter happy + _ = pp + _ = vv +} + +func PanicOn(err error) { + if err != nil { + panic(err) + } +} + +func PP(format string, a ...interface{}) { + if VerboseVerbose { + TSPrintf(format, a...) + } +} + +func VV(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +func AlwaysPrintf(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +var tsPrintfMut sync.Mutex + +// time-stamped printf +func TSPrintf(format string, a ...interface{}) { + tsPrintfMut.Lock() + Printf("\n%s %s ", FileLine(3), ts()) + Printf(format+"\n", a...) + tsPrintfMut.Unlock() +} + +// get timestamp for logging purposes +func ts() string { + return time.Now().Format(RFC3339UsecTz0) +} + +// so we can multi write easily, use our own printf +var OurStdout io.Writer = os.Stdout + +// Printf formats according to a format specifier and writes to standard output. +// It returns the number of bytes written and any write error encountered. +func Printf(format string, a ...interface{}) (n int, err error) { + return fmt.Fprintf(OurStdout, format, a...) +} + +func FileLine(depth int) string { + _, fileName, fileLine, ok := runtime.Caller(depth) + var s string + if ok { + s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine) + } else { + s = "" + } + return s +} + +func stack() string { + return string(debug.Stack()) +} + +var _ = stack + +func FileExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return false + } + return true +} + +func DirExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return true + } + return false +} + +func FileSize(name string) (int64, error) { + fi, err := os.Stat(name) + if err != nil { + return -1, err + } + return fi.Size(), nil +} + +// Caller returns the name of the calling function. +func Caller(upStack int) string { + // elide ourself and runtime.Callers + target := upStack + 2 + + pc := make([]uintptr, target+2) + n := runtime.Callers(0, pc) + + f := runtime.Frame{Function: "unknown"} + if n > 0 { + frames := runtime.CallersFrames(pc[:n]) + for i := 0; i <= target; i++ { + contender, more := frames.Next() + if i == target { + f = contender + } + if !more { + break + } + } + } + return f.Function +} diff --git a/test/holder.go b/test/holder.go index ed91d669b..6deb44ce5 100644 --- a/test/holder.go +++ b/test/holder.go @@ -160,7 +160,7 @@ func (h *Holder) SetBitTime(index, field string, rowID, columnID uint64, t *time } shard := columnID / pilosa.ShardWidth - tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard}) + tx := idx.Index.Txf().NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard}) defer tx.Rollback() _, err = f.SetBit(tx, rowID, columnID, t) @@ -179,7 +179,7 @@ func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) { } shard := columnID / pilosa.ShardWidth - tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard}) + tx := idx.Index.Txf().NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard}) defer tx.Rollback() _, err = f.ClearBit(tx, rowID, columnID) @@ -205,7 +205,7 @@ func (h *Holder) SetValue(index, field string, columnID uint64, value int64) *In panic(err) } shard := columnID / pilosa.ShardWidth - tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard}) + tx := idx.Index.Txf().NewTx(pilosa.Txo{Write: true, Index: idx.Index, Shard: shard}) defer tx.Rollback() _, err = f.SetValue(tx, columnID, value) @@ -226,7 +226,7 @@ func (h *Holder) Value(index, field string, columnID uint64) (int64, bool) { panic(err) } shard := columnID / pilosa.ShardWidth - tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: false, Index: idx.Index, Shard: shard}) + tx := idx.Index.Txf().NewTx(pilosa.Txo{Write: false, Index: idx.Index, Shard: shard}) defer tx.Rollback() val, exists, err := f.Value(tx, columnID) diff --git a/test/index.go b/test/index.go index b78199263..376ee6d65 100644 --- a/test/index.go +++ b/test/index.go @@ -59,7 +59,7 @@ func (i *Index) Reopen() error { if err := i.Index.Close(); err != nil { return err } - return i.Index.Open(false) + return i.Index.Open() } // CreateField creates a field with the given options. diff --git a/tx.go b/tx.go index 6d2b0642c..8546ac8d3 100644 --- a/tx.go +++ b/tx.go @@ -68,6 +68,11 @@ type Tx interface { // Commit makes the updates in the Tx visible to subsequent transactions. Commit() error + // IsDone must return true if Rollback() or Commit() has already + // been called. Otherwise it must return false. This allows + // DBWrapper.CleanupTx(tx Tx) to be idempotent. + IsDone() bool + // Readonly returns the flag this transaction was created with // during NewTx. If the transaction is writable, it will return false. Readonly() bool @@ -197,7 +202,7 @@ type Tx interface { Group() *TxGroup // Dump is for debugging, what does this Tx see as its database? - Dump() + Dump(short bool) // Options returns the options used to create this Tx. This // can be implementd by embedding Txo, and Txo provides the diff --git a/txfactory.go b/txfactory.go index 6eddbe19e..95c8ccf7d 100644 --- a/txfactory.go +++ b/txfactory.go @@ -37,6 +37,7 @@ const ( RoaringTxn string = "roaring" LmdbTxn string = "lmdb" RBFTxn string = "rbf" + BadgerTxn string = "badger" ) // DefaultTxsrc is set here. pilosa/server/config.go references it @@ -389,6 +390,10 @@ type TxFactory struct { holder *Holder blueGreenReg *blueGreenRegistry + + // allow holder to activate blue-green checking only + // once we have synced both sides at start up time. + blueGreenOff bool } func (f *TxFactory) Types() []txtype { @@ -403,20 +408,41 @@ const ( roaringTxn txtype = 1 // these don't really have any transactions rbfTxn txtype = 2 lmdbTxn txtype = 3 + badgerTxn txtype = 4 ) +// these need to be skipped by the holder.go field scanner that +// calls IsTxDatabasePath +var allTypesWithSuffixes = []txtype{rbfTxn, lmdbTxn, badgerTxn} + +// FileSuffix is used to determine backend directory names. +// We append '@' to be sure we never collide with a field name +// inside the index directory. In the future for different +// versions of the same backend, there might be version +// identifier tacked on too. func (ty txtype) FileSuffix() string { switch ty { case roaringTxn: return "" case rbfTxn: - return "-rbfdb" + return "-rbfdb@" case lmdbTxn: - return "-lmdb" + return "-lmdb@" + case badgerTxn: + return "-badgerdb@" } panic(fmt.Sprintf("unkown txtype %v", int(ty))) } +func (txf *TxFactory) IsTxDatabasePath(path string) bool { + for _, ty := range allTypesWithSuffixes { + if strings.HasSuffix(path, ty.FileSuffix()) { + return true + } + } + return false +} + func (txf *TxFactory) NeedsSnapshot() (b bool) { for _, ty := range txf.types { switch ty { @@ -447,6 +473,8 @@ func MustTxsrcToTxtype(txsrc string) (types []txtype) { types = append(types, rbfTxn) case LmdbTxn: // "lmdb" types = append(types, lmdbTxn) + case BadgerTxn: // "badger" + types = append(types, badgerTxn) default: panic(fmt.Sprintf("unknown txsrc '%v'", s)) } @@ -467,12 +495,15 @@ func NewTxFactory(txsrc string, holderDir string, holder *Holder) (f *TxFactory, types := MustTxsrcToTxtype(txsrc) f = &TxFactory{ - types: types, - typeOfTx: txsrc, - holder: holder, - blueGreenReg: newBlueGreenReg(), + types: types, + typeOfTx: txsrc, + holder: holder, + } + if len(types) == 2 { + f.blueGreenReg = newBlueGreenReg(types) } f.dbPerShard = f.NewDBPerShard(types, holderDir) + return f, err } @@ -489,6 +520,8 @@ type Txo struct { per *DBPerShard Group *TxGroup + + blueGreenOff bool } func (o Txo) String() string { @@ -528,7 +561,7 @@ func (f *TxFactory) CloseIndex(idx *Index) error { return nil } -func (f *TxFactory) CloseDB() (err error) { +func (f *TxFactory) Close() (err error) { if f.dbsClosed { return nil } @@ -684,7 +717,7 @@ func (g *TxGroup) AbortGroup() { } func (f *TxFactory) NewTx(o Txo) (txn Tx) { - f.mu.Lock() // deadlock here + f.mu.Lock() defer f.mu.Unlock() defer func() { @@ -693,6 +726,8 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) { } }() + o.blueGreenOff = f.blueGreenOff + indexName := "" if o.Index != nil { indexName = o.Index.name @@ -711,8 +746,15 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) { // per-shard database. Opens a new one if needed. dbs, err := f.dbPerShard.GetDBShard(indexName, o.Shard, o.Index) panicOn(err) + + if dbs.Shard != o.Shard { + panic(fmt.Sprintf("asked for o.Shard=%v but got dbs.Shard=%v", int(o.Shard), int(dbs.Shard))) + } + //vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.types='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.types, dbs.W) + o.dbs = dbs // our specific database per shard. o.per = f.dbPerShard // for top level debug Dumps + tx, err := dbs.NewTx(o.Write, indexName, o) if err != nil { panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored")) @@ -730,6 +772,8 @@ func (ty txtype) String() string { return "rbfTxn" case lmdbTxn: return "lmdbTxn" + case badgerTxn: + return "badgerTxn" } panic(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty))) } @@ -767,7 +811,7 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64, // hashOnly means only show the value hash, not the content bits. // showOps means display the ops log. -func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool) (r string) { +func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool, o Txo) (r string) { paths, err := listFilesUnderDir(idx.path, false, "", true) panicOn(err) index := idx.name @@ -784,9 +828,9 @@ func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool) (r string) { s, _, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard, showOps, hashOnly, os.Stdout) panicOn(err) //r += fmt.Sprintf("path:'%v' fragment contains:\n") + s - if s == "" { - s = "" - } + //if s == "" { + //s = "" + //} r += s n++ } @@ -1073,3 +1117,154 @@ func printContainers(w io.Writer, info roaring.BitmapInfo, pC pointerContext) { } tw.Flush() } + +var _ = anyGlobalDBWrappersStillOpen // happy linter + +func anyGlobalDBWrappersStillOpen() bool { + if globalRoaringReg.Size() != 0 { + return true + } + if globalRbfDBReg.Size() != 0 { + return true + } + if globalLMDBReg.Size() != 0 { + return true + } + if globalBadgerReg.Size() != 0 { + return true + } + return false +} + +func (f *TxFactory) blueGreenOnIfRunningBlueGreen() { + if len(f.types) == 2 { + f.blueGreenOff = false + } +} + +func (f *TxFactory) blueGreenOffIfRunningBlueGreen() { + if len(f.types) == 2 { + f.blueGreenOff = true + } +} + +func (f *TxFactory) hasRoaring() bool { + return f.types[0] == roaringTxn || f.types[1] == roaringTxn +} + +var _ = (&TxFactory{}).hasRoaring // happy linter + +func (f *TxFactory) blueHasData() (hasData bool, err error) { + if len(f.types) != 2 { + return false, nil + } + return f.dbPerShard.HasData(0) +} + +// green2blue is called at the very end of Holder.Open(), so +// we know that the holder is ready to go, knowing its holder.Indexes(), fields, +// view, shards, and other metadata if any. +// +// Called by test Test_TxFactory_UpdateBlueFromGreen_OnStartup() in +// txfactory_internal_test.go as well. +// +func (f *TxFactory) green2blue(holder *Holder) (err error) { + + // Holder.Open will always call us, even without blue_green. Which is fine. + // We are just a no-op in that case. + if len(f.types) != 2 { + return nil + } + + blueDest := f.types[0] + greenSrc := f.types[1] + idxs := holder.Indexes() + + verifyInsteadOfCopy := false + + hasData, err := f.blueHasData() + if err != nil { + return errors.Wrap(err, "TxFactory.green2blue DataSize(0)") + } + + if hasData { + verifyInsteadOfCopy = true + } + + for _, idx := range idxs { + + blueShards, err := TypedDBPerShardGetLocalShardsForIndex(blueDest, idx, "") + if err != nil { + return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v') error fetching blueShards", idx.name)) + } + greenShards, err := TypedDBPerShardGetLocalShardsForIndex(greenSrc, idx, "") + if err != nil { + return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v') error fetching greenShards", idx.name)) + } + + diff := f.shardSliceDiff(blueShards, greenShards) + if diff != "" { + return fmt.Errorf("blue[%v] and green[%v] have different shards for index '%v': %v", blueDest, greenSrc, idx.name, diff) + } + + shards := idx.AvailableShards(localOnly).Slice() + + diff2 := f.shardSliceDiff(blueShards, shards) + if diff2 != "" { + return fmt.Errorf("blue[%v] and meta data (from green[%v]?)have different shards for index '%v': %v", blueDest, greenSrc, idx.name, diff2) + } + + for _, shard := range shards { + + dbs, err := f.dbPerShard.GetDBShard(idx.name, shard, idx) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v', shard='%v')", idx.name, int(shard))) + } + + if verifyInsteadOfCopy { + // verify all containers + err = dbs.verifyBlueEqualsGreen(&holder.numCtBlueGreenVerified) + if err != nil { + return errors.Wrap(err, + fmt.Sprintf("dbs.verifyBlueEqualsGreen(blue='%v', "+ + "green='%v') for index='%v', shard='%v'", + blueDest, greenSrc, idx.name, int(shard))) + } + } else { + // the main copy work + err = dbs.populateBlueFromGreen() + if err != nil { + return errors.Wrap(err, + fmt.Sprintf("dbs.copyGreenToBlue(blue='%v', "+ + "green='%v') for index='%v', shard='%v'", + blueDest, greenSrc, idx.name, int(shard))) + } + } + } + } + return nil +} + +func (f *TxFactory) shardSliceDiff(blueShards, greenShards []uint64) (diff string) { + nb := len(blueShards) + ng := len(greenShards) + if nb != ng { + diff = fmt.Sprintf("blueShard[%v] count = %v; greenShard[%v] count = %v; ", f.types[0], nb, f.types[1], ng) + } + b := make(map[uint64]bool) + g := make(map[uint64]bool) + for _, bs := range blueShards { + b[bs] = true + } + for _, gs := range greenShards { + g[gs] = true + } + bmg := mapDiff(b, g) // get blue - green + gmb := mapDiff(g, b) // get green - blue + + if len(bmg) == 0 && len(gmb) == 0 { + return "" + } + diff += fmt.Sprintf("shard diff: blueMinusGreen shards: '%#v'; greenMinusBlue shards: '%#v'", bmg, gmb) + return +} diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go index 4cac2252e..235a009a6 100644 --- a/txfactory_internal_test.go +++ b/txfactory_internal_test.go @@ -15,6 +15,7 @@ package pilosa import ( + "context" "fmt" "os" "testing" @@ -25,10 +26,10 @@ import ( func Test_TxFactory_Qcx_query_context(t *testing.T) { src := os.Getenv("PILOSA_TXSRC") - if src == "rbf" || src == "lmdb" { + if src == "rbf" || src == "lmdb" || src == "badger" { // ok } else { - t.Skip("this test only for lmdb and rbf") + t.Skip("this test only for lmdb and rbf and badger") } shard := uint64(0) @@ -50,7 +51,7 @@ func Test_TxFactory_Qcx_query_context(t *testing.T) { default: } // add to the group txn on the txf. - qcx := idx.Txf.NewQcx() + qcx := idx.holder.txf.NewQcx() tx, finisher := qcx.GetTx(Txo{Write: true, Index: idx, Shard: f.shard}) @@ -95,8 +96,256 @@ func Test_TxFactory_Qcx_query_context(t *testing.T) { // we will crash as the goroutines making Tx will try to use the env // after it is closed. It can take quite a while. // one writer might be blocking the other... so ask for only N-1 at first. - //barrier.BlockUntil(N - 1) - barrier.BlockUntil(N) - //barrier.UnblockReaders() - //time.Sleep(1 * time.Second) + barrier.BlockUntil(N - 1) + //barrier.BlockUntil(N) + barrier.UnblockReaders() + time.Sleep(1 * time.Second) +} + +// test TxFactory.green2blue +// +// blue_green starting with an empty or full blue database +// should copy all of green (if blue is empty); or if blue is ull, +// verify that blue has all the same bits as green. +// +// Benefits: a) we start with known identical state so our testing/comparisons can be valid; +// and b) we have an easy migration mechanism, to go from one storage format to another. +// +func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) { + + orig := os.Getenv("PILOSA_TXSRC") + defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests! + + checked := []string{"lmdb", "roaring", "badger", "rbf"} + + for _, blue := range checked { + for _, green := range checked { + if blue == green { + continue + } + blue_green := blue + "_" + green + + // ============================= + // Begin setup. + // + // Setup happens with green only. + + os.Setenv("PILOSA_TXSRC", green) + h, path, err := makeHolder(t) + if err != nil { + t.Fatalf("creating holder: %v", err) + } + defer os.RemoveAll(path) + + // we will manually h.Close() below + + // Write bits to separate indexes. + testSetBit(t, h, "i0", "f", 100, 200) + testSetBit(t, h, "i1", "f", 100, 200) + testSetBit(t, h, "i1", "f", 100, 12345678) + + testOp := testHolderOperator{} + ctx := context.Background() + err = h.Process(ctx, &testOp) + if err != nil { + t.Fatalf("processing holder: %v", err) + } + expected := testHolderOperator{ + indexSeen: 2, indexProcessed: 2, + fieldSeen: 2, fieldProcessed: 2, + viewSeen: 2, viewProcessed: 2, + fragmentSeen: 3, fragmentProcessed: 3, + } + if testOp != expected { + t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp) + } + + // verify data is there + rowID := uint64(100) + colID := uint64(200) + _, _ = rowID, colID + testMustHaveBit(t, h, "i0", "f", rowID, colID) + testMustHaveBit(t, h, "i1", "f", 100, 200) + testMustHaveBit(t, h, "i1", "f", 100, 12345678) + + h.Close() + + // can we re.Open the same holder h? hopefully without a problem. + panicOn(h.Open()) + + testMustHaveBit(t, h, "i0", "f", rowID, colID) + testMustHaveBit(t, h, "i1", "f", 100, 200) + testMustHaveBit(t, h, "i1", "f", 100, 12345678) + h.Close() + + // successful re-open and then Close again of h. + + // check that we can open a NewHolder on green, on same path, and still see our bits. + // Because the NewHolder is the code that creates and configures TxFactory as blue_green. + h2 := NewHolder(path, nil) + panicOn(h2.Open()) + + testMustHaveBit(t, h2, "i0", "f", rowID, colID) + testMustHaveBit(t, h2, "i1", "f", 100, 200) + testMustHaveBit(t, h2, "i1", "f", 100, 12345678) + h2.Close() + + // verify that blue does not have it. + // open a new holder on path, just looking at blue. + os.Setenv("PILOSA_TXSRC", blue) + + h3 := NewHolder(path, nil) + panicOn(h3.Open()) + + testMustNotHaveBit(t, h3, "i0", "f", rowID, colID) + testMustNotHaveBit(t, h3, "i1", "f", 100, 200) + testMustNotHaveBit(t, h3, "i1", "f", 100, 12345678) + + h3.Close() + + // ============================= + // Setup done. On to actual test. + + // Opening in blue_green mode means that once Holder.Open() + // returns without error, the blue and green databases are + // identical. + // Since blue is empty, the blue database will get synched up + // with the green during Holder.Open(). + + os.Setenv("PILOSA_TXSRC", blue_green) + + // open a holder with path again, now looking at both blue and green. + // The Holder.Open should do the migration from green, populating blue. + h4 := NewHolder(path, nil) + panicOn(h4.Open()) + defer h4.Close() + + testMustHaveBit(t, h4, "i0", "f", rowID, colID) + testMustHaveBit(t, h4, "i1", "f", 100, 200) + testMustHaveBit(t, h4, "i1", "f", 100, 12345678) + } + } +} + +// test the situation where we startup blue_green with existing data and +// go to verify it but blue has more data than green. +// That will also cause query divergence. +func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) { + + orig := os.Getenv("PILOSA_TXSRC") + defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests! + + checked := []string{"lmdb", "roaring", "badger", "rbf"} + + for _, blue := range checked { + for _, green := range checked { + if blue == green { + continue + } + blue_green := blue + "_" + green + + // ============================= + // Begin setup. + // + // Setup happens with green only. + + os.Setenv("PILOSA_TXSRC", green) + h, path, err := makeHolder(t) + if err != nil { + t.Fatalf("creating holder: %v", err) + } + defer os.RemoveAll(path) + + // we will manually h.Close() below + + // Write bits to separate indexes. + testSetBit(t, h, "i0", "f", 100, 200) + testSetBit(t, h, "i1", "f", 100, 200) + testSetBit(t, h, "i1", "f", 100, 12345678) + + testOp := testHolderOperator{} + ctx := context.Background() + err = h.Process(ctx, &testOp) + if err != nil { + t.Fatalf("processing holder: %v", err) + } + expected := testHolderOperator{ + indexSeen: 2, indexProcessed: 2, + fieldSeen: 2, fieldProcessed: 2, + viewSeen: 2, viewProcessed: 2, + fragmentSeen: 3, fragmentProcessed: 3, + } + if testOp != expected { + t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp) + } + + // verify data is there + rowID := uint64(100) + colID := uint64(200) + _, _ = rowID, colID + testMustHaveBit(t, h, "i0", "f", rowID, colID) + testMustHaveBit(t, h, "i1", "f", 100, 200) + testMustHaveBit(t, h, "i1", "f", 100, 12345678) + + h.Close() + + // verify that blue does not have it. + // open a new holder on path, just looking at blue. + os.Setenv("PILOSA_TXSRC", blue) + + h3 := NewHolder(path, nil) + panicOn(h3.Open()) + + testMustNotHaveBit(t, h3, "i0", "f", rowID, colID) + testMustNotHaveBit(t, h3, "i1", "f", 100, 200) + testMustNotHaveBit(t, h3, "i1", "f", 100, 12345678) + + h3.Close() + + // ============================= + // Setup done. On to actual test. + + // Opening in blue_green mode means that once Holder.Open() + // returns without error, the blue and green databases are + // identical. + // Since blue is empty, the blue database will get synched up + // with the green during Holder.Open(). + + os.Setenv("PILOSA_TXSRC", blue_green) + + // open a holder with path again, now looking at both blue and green. + // The Holder.Open should do the migration from green, populating blue. + h4 := NewHolder(path, nil) + panicOn(h4.Open()) + + testMustHaveBit(t, h4, "i0", "f", rowID, colID) + testMustHaveBit(t, h4, "i1", "f", 100, 200) + testMustHaveBit(t, h4, "i1", "f", 100, 12345678) + h4.Close() + + // now open just blue, and add a bit to a new index, i2. + os.Setenv("PILOSA_TXSRC", blue) + + h5 := NewHolder(path, nil) + panicOn(h5.Open()) + testSetBit(t, h5, "i2", "f", 500, 777) + h5.Close() + + // now open blue_green. should get a verification failure + // due to the extra bit in blue. + os.Setenv("PILOSA_TXSRC", blue_green) + + // BEGIN verficiation that should ERROR out b/c blue has more data. + + // open a holder with path again, now looking at both blue and green. + // The Holder.Open should verify blue against green and notice the extra bit. + h6 := NewHolder(path, nil) + err = h6.Open() + if err == nil { + h6.Close() + t.Fatalf("should have had blue-green verification fail on Holder.Open") + } + h6.Close() + } + } } diff --git a/util.go b/util.go index 0e243c55a..25ed3f826 100644 --- a/util.go +++ b/util.go @@ -20,8 +20,12 @@ import ( "fmt" "io/ioutil" "net" + "os" + "path/filepath" "reflect" "sort" + "strings" + "syscall" "unsafe" "github.com/pilosa/pilosa/v2/roaring" @@ -237,3 +241,119 @@ func fromInterval16(a []roaring.Interval16) []byte { } return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4] } + +// DiskUse reports the total bytes uses by all files under root +// that match requiredSuffix. requiredSuffix can be empty string. +// Space used by directories is not counted. +func DiskUse(root string, requiredSuffix string) (tot int, err error) { + if !DirExists(root) { + return -1, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root) + } + + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if info == nil { + panic(fmt.Sprintf("info was nil for path = '%v'", path)) + } + if info.IsDir() { + // skip the size of directories themselves, only summing files. + } else { + sz := info.Size() + if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) { + tot += int(sz) + } + } + return nil + }) + return +} + +// rootDir must exist. Return the size in bytes of the largest sub-directory +// that has the required suffix. The largestSize is from DiskUse() called +// on the sub-dir. DiskUse only counts file size, nothing for directory inodes. +func SubdirLargestDirWithSuffix(rootDir, requiredDirSuffix string) (exists bool, largestSize int, err error) { + if !DirExists(rootDir) { + return false, -1, fmt.Errorf("SubdirExistsWithSuffix error: root directory '%v' not found", rootDir) + } + + err = filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { + if info == nil { + panic(fmt.Sprintf("info was nil for path = '%v'", path)) + } + + if info.IsDir() && strings.HasSuffix(path, requiredDirSuffix) { + exists = true + size, err := DiskUse(path, "") + if err != nil { + // disk error? report it + return err + } + if size > largestSize { + largestSize = size + } + } + return nil + }) + if err != nil { + return exists, -1, err + } + return +} + +// called by Holder.hasRoaringData() +func roaringFragmentHasData(path string, index, field, view string, shard uint64) (hasData bool, err error) { + + var info roaring.BitmapInfo + _ = info + var f *os.File + f, err = os.Open(path) + if err != nil { + return + } + + var fi os.FileInfo + fi, err = f.Stat() + if err != nil { + return + } + + // Memory map the file. + data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + err = errors.Wrap(err, "mmapping") + return + } + defer func() { + err = syscall.Munmap(data) + if err != nil { + err = errors.Wrap(err, "roaringFragmentHasData: munmap failed") + } + err = f.Close() + if err != nil { + err = errors.Wrap(err, "roaringFragmentHasData f.Close() in defer") + } + }() + + // Attach the mmap file to the bitmap. + var rbm *roaring.Bitmap + rbm, _, err = roaring.InspectBinary(data, true, &info) + if err != nil { + err = errors.Wrap(err, "inspecting") + return + } + + if info.ContainerCount > 0 { + return true, nil + } + if info.Ops > 0 { + return true, nil + } + + citer, found := rbm.Containers.Iterator(0) + _ = found + + for citer.Next() { + return true, nil + } + + return +} diff --git a/utils_internal_test.go b/utils_internal_test.go index 15d3ac9ee..91f2c0745 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -167,9 +167,9 @@ func (t *ClusterCluster) SetBit(index, field string, rowID, colID uint64, x *tim } if err := func() error { - idx := c.holder.indexes[f.index] + idx := c.holder.Index(f.index) shard := colID / ShardWidth - tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) + tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard}) if tx != nil { defer tx.Rollback() } @@ -488,11 +488,11 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error // there will be two -badgerdb directories/databases, we need to copy // from src to dest the fragment. This simulates sending the fragment over the network. srcIdx := srcCluster.holder.Index(src.Index) - srctx := srcIdx.Txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment, Shard: srcFragment.shard}) + srctx := srcIdx.holder.txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment, Shard: srcFragment.shard}) destIdx := destCluster.holder.Index(src.Index) - desttx := destIdx.Txf.NewTx(Txo{Write: writable, Index: destIdx, Fragment: destFragment, Shard: destFragment.shard}) + desttx := destIdx.holder.txf.NewTx(Txo{Write: writable, Index: destIdx, Fragment: destFragment, Shard: destFragment.shard}) citer, _, err := srctx.ContainerIterator(src.Index, src.Field, src.View, src.Shard, 0) panicOn(err) diff --git a/view.go b/view.go index 1a10ac5b6..52c940986 100644 --- a/view.go +++ b/view.go @@ -173,7 +173,6 @@ var workQueue = make(chan struct{}, runtime.NumCPU()*2) // replaces v.openFragments() with Tx generic code. func (v *view) openFragmentsInTx() error { - // we think this is correct for dbpershard, but might be slower. TODO shards, err := DBPerShardGetShardsForIndex(v.idx, v.path) if err != nil { return errors.Wrap(err, "DBPerShardGetShardsForIndex()") @@ -368,15 +367,6 @@ func (v *view) notifyIfNewShard(shard uint64) { func (v *view) newFragment(path string, shard uint64) *fragment { - if v.holder != nil && v.idx != nil { - // A view must have its v.idx *Index registered with its holder. - // Otherwise TestField_AvailableShards crashes, as one example. - hIdx := v.holder.Index(v.idx.name) - if hIdx == nil && v.idx != nil { - v.holder.addIndexFromField(v.idx) - } - } - frag := newFragment(v.holder, path, v.index, v.field, v.name, shard, v.flags()) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize @@ -402,7 +392,7 @@ func (v *view) deleteFragment(shard uint64) error { idx := f.holder.Index(v.index) f.Close() - if err := idx.Txf.DeleteFragmentFromStore(f.index, f.field, f.view, f.shard, f); err != nil { + if err := idx.holder.txf.DeleteFragmentFromStore(f.index, f.field, f.view, f.shard, f); err != nil { return errors.Wrap(err, "DeleteFragment") } delete(v.fragments, shard) @@ -418,7 +408,7 @@ func (v *view) row(txOrig Tx, rowID uint64) (*Row, error) { tx := txOrig if NilInside(tx) { - tx = v.idx.Txf.NewTx(Txo{Write: !writable, Index: v.idx, Fragment: frag, Shard: frag.shard}) + tx = v.idx.holder.txf.NewTx(Txo{Write: !writable, Index: v.idx, Fragment: frag, Shard: frag.shard}) defer tx.Rollback() } @@ -445,7 +435,7 @@ func (v *view) setBit(txOrig Tx, rowID, columnID uint64) (changed bool, err erro tx := txOrig if NilInside(tx) { - tx = v.idx.Txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard}) + tx = v.idx.holder.txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard}) defer func() { if err == nil { panicOn(tx.Commit()) @@ -467,7 +457,7 @@ func (v *view) clearBit(txOrig Tx, rowID, columnID uint64) (changed bool, err er tx := txOrig if NilInside(tx) { - tx = v.idx.Txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard}) + tx = v.idx.holder.txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard}) defer func() { if err == nil { panicOn(tx.Commit()) @@ -490,7 +480,7 @@ func (v *view) value(txOrig Tx, columnID uint64, bitDepth uint) (value int64, ex tx := txOrig if NilInside(tx) { - tx = frag.idx.Txf.NewTx(Txo{Write: !writable, Index: frag.idx, Fragment: frag, Shard: frag.shard}) + tx = frag.idx.holder.txf.NewTx(Txo{Write: !writable, Index: frag.idx, Fragment: frag, Shard: frag.shard}) defer tx.Rollback() } @@ -507,7 +497,7 @@ func (v *view) setValue(txOrig Tx, columnID uint64, bitDepth uint, value int64) tx := txOrig if NilInside(tx) { - tx = v.idx.Txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard}) + tx = v.idx.holder.txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard}) defer func() { if err == nil { panicOn(tx.Commit()) @@ -530,7 +520,7 @@ func (v *view) clearValue(txOrig Tx, columnID uint64, bitDepth uint, value int64 tx := txOrig if NilInside(tx) { - tx = v.idx.Txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard}) + tx = v.idx.holder.txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard}) defer func() { if err == nil { panicOn(tx.Commit()) diff --git a/view_internal_test.go b/view_internal_test.go index 759167458..9076fb788 100644 --- a/view_internal_test.go +++ b/view_internal_test.go @@ -36,6 +36,7 @@ func mustOpenView(tb testing.TB, index, field, name string) *view { h := NewHolder(path, nil) // h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment + idx, err := h.createIndex(index, IndexOptions{}) testhook.Cleanup(tb, func() { h.Close()