diff --git a/.circleci/config.yml b/.circleci/config.yml index 699be66ac..f0ec335ba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -208,7 +208,7 @@ workflows: - setup matrix: parameters: - test_make_target: ["test-race", "test-txstore-rbf", "test-txstore-rbf_lmdb"] + test_make_target: ["test-race", "test-txstore-rbf", "test-txstore-rbf_bolt"] - test: name: test-shardwidth-22 shard_width: "22" diff --git a/Makefile b/Makefile index d6e1caec8..dad5fec72 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf_lmdb test-txstore-rbf +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf CLONE_URL=github.com/pilosa/pilosa MOD_VERSION=v2 @@ -309,6 +309,6 @@ install-gometalinter: test-txstore-rbf: PILOSA_TXSRC=rbf $(MAKE) testv-race -test-txstore-rbf_lmdb: - PILOSA_TXSRC=rbf_lmdb $(MAKE) testv-race +test-txstore-rbf_bolt: + PILOSA_TXSRC=rbf_bolt $(MAKE) testv-race diff --git a/bolt.go b/bolt.go index 7980bb3a7..4d7830a44 100644 --- a/bolt.go +++ b/bolt.go @@ -37,6 +37,8 @@ import ( bolt "go.etcd.io/bbolt" ) +const isDebugRun = false + // boltRegistrar facilitates shutdown // of all the bolt databases started under // tests. Its needed because most tests don't cleanup @@ -1419,6 +1421,21 @@ func (tx *BoltTx) toContainer(typ byte, v []byte) (r *roaring.Container) { return ToContainer(typ, w) } +func ToContainer(typ byte, w []byte) (c *roaring.Container) { + switch typ { + case roaring.ContainerArray: + c = roaring.NewContainerArray(toArray16(w)) + case roaring.ContainerBitmap: + c = roaring.NewContainerBitmap(-1, toArray64(w)) + case roaring.ContainerRun: + c = roaring.NewContainerRun(toInterval16(w)) + default: + panic(fmt.Sprintf("unknown container: %v", typ)) + } + c.SetMapped(true) + return c +} + // StringifiedBoltKeys returns a string with all the container // keys available in bolt. func (w *BoltWrapper) StringifiedBoltKeys(optionalUseThisTx Tx, short bool) (r string) { diff --git a/const_amd64.go b/const_amd64.go new file mode 100644 index 000000000..5298800ff --- /dev/null +++ b/const_amd64.go @@ -0,0 +1,19 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build amd64 + +package pilosa + +const TxInitialMmapSize = 4 << 30 // 4GB diff --git a/const_other.go b/const_other.go new file mode 100644 index 000000000..7e99c229e --- /dev/null +++ b/const_other.go @@ -0,0 +1,21 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !amd64 + +package pilosa + +// this is a stubbed out file to let 386/arm build. + +const TxInitialMmapSize = 1 << 30 // 1GB diff --git a/dbshard.go b/dbshard.go index 74d36c771..2c9e2989e 100644 --- a/dbshard.go +++ b/dbshard.go @@ -478,7 +478,6 @@ func (dbs *DBShard) DumpAll() { switch ty { case roaringTxn: case rbfTxn: - case lmdbTxn: case boltTxn: default: panic(fmt.Sprintf("unknown txtyp: '%v'", ty)) @@ -605,8 +604,6 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs * registry = globalRoaringReg case rbfTxn: registry = globalRbfDBReg - case lmdbTxn: - registry = globalLMDBReg case boltTxn: registry = globalBoltReg default: @@ -888,9 +885,9 @@ func listDirUnderDir(root string, includeRoot bool, requiredSuffix string, ignor // The blue is the destination -- this is always types[0]. // The green source is always types[1]. The mnemonic is blue_geen. // The blue is first, so it is in types[0]. The green -// is second, in types[1]. For example, with PILOSA_TXSRC=lmdb_roaring -// we have lmdb as blue, and roaring as green. The contents of -// lmdb must be empty or exactly match roaring. If lmdb +// is second, in types[1]. For example, with PILOSA_TXSRC=bolt_roaring +// we have bolt as blue, and roaring as green. The contents of +// bolt must be empty or exactly match roaring. If bolt // starts empty, it will be populated from roaring by // populateBlueFromGreen(). // diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go index b58f86045..02340035c 100644 --- a/dbshard_internal_test.go +++ b/dbshard_internal_test.go @@ -74,7 +74,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { orig := os.Getenv("PILOSA_TXSRC") defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests! - for _, src := range []string{"lmdb", "roaring", "bolt", "rbf"} { + for _, src := range []string{"roaring", "bolt", "rbf"} { os.Setenv("PILOSA_TXSRC", src) @@ -131,14 +131,6 @@ rick/_exists/views/standard/fragments/217 rick/_exists/views/standard/fragments/93 rick/_exists/views/standard/fragments/219 rick/_exists/views/standard/fragments/223 -`, - "lmdb": ` -rick.index.txstores@@@/store-lmdb@@/shard.0093-lmdb@ -rick.index.txstores@@@/store-lmdb@@/shard.0215-lmdb@ -rick.index.txstores@@@/store-lmdb@@/shard.0217-lmdb@ -rick.index.txstores@@@/store-lmdb@@/shard.0219-lmdb@ -rick.index.txstores@@@/store-lmdb@@/shard.0221-lmdb@ -rick.index.txstores@@@/store-lmdb@@/shard.0223-lmdb@ `, "bolt": ` rick.index.txstores@@@/store-boltdb@@/shard.0093-boltdb@/bolt.db @@ -173,13 +165,6 @@ func makeSampleRoaringDir(root, txsrc string, minBytes int, h *Holder) { shard = shards[i] } switch txsrc { - case "lmdb": - makeLMDBtestDB(root+sep+fn, h, shard) - // also have to make the DBShard in our in-memory tree, - // or else the search won't find it because - // DBPerShard won't know anything about it. - helperCreateDBShard(h, index, shard) - continue case "bolt": makeBolttestDB(root+sep+fn, h, shard) helperCreateDBShard(h, index, shard) @@ -210,14 +195,6 @@ func helperCreateDBShard(h *Holder, index string, shard uint64) { _ = dbs } -func makeLMDBtestDB(path string, h *Holder, shard uint64) { - i := uint64(1) - w, _ := mustOpenEmptyLMDBWrapper(path) - LMDBMustSetBitvalue(w, "index", "field", "view", shard, i) - w.Close() - -} - func makeBolttestDB(path string, h *Holder, shard uint64) { i := uint64(1) w, _ := mustOpenEmptyBoltWrapper(path) diff --git a/dbshard_test.go b/dbshard_test.go index 9912e03d9..dc58efd5e 100644 --- a/dbshard_test.go +++ b/dbshard_test.go @@ -17,7 +17,6 @@ package pilosa_test import ( "context" "fmt" - "os" "reflect" "testing" @@ -28,59 +27,6 @@ import ( "github.com/pilosa/pilosa/v2/test" ) -func skipForNonLMDB(t *testing.T) { - src := os.Getenv("PILOSA_TXSRC") - if src != "lmdb" { - t.Skip("skip if not lmdb") - } -} - -var _ = skipForNonLMDB // happy linter - -// Can't write it all to one shard like we do (did). -func Test_DBPerShard_multiple_shards_used(t *testing.T) { - skipForNonLMDB(t) - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := c.GetHolder(0) - index := "i" - hldr.SetBit(index, "general", 10, 0) - hldr.SetBit(index, "general", 10, ShardWidth+1) - hldr.SetBit(index, "general", 10, ShardWidth+2) - - hldr.SetBit(index, "general", 11, 2) - hldr.SetBit(index, "general", 11, ShardWidth+2) - - types := pilosa.MustTxsrcToTxtype("lmdb") - idx := hldr.Index(index) - shardsU := []uint64{0, 1, 2} - pathShard := []string{} - - // check that 3 different shard databases/files were made - for i := 0; i < 2; i++ { - - path, err := hldr.Txf().GetDBShardPath(index, shardsU[i], idx, types[0], !writable) - panicOn(err) - pathShard = append(pathShard, path) - - if !DirExists(pathShard[i]) { - panic(fmt.Sprintf("no shard made for pathShard[%v]='%v'", i, pathShard[i])) - } - sz, err := pilosa.DiskUse(pathShard[i], "") - panicOn(err) - - if sz < 100 { - panic(fmt.Sprintf("shard %v was too small", i)) - } - } - - if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: `Union(Row(general=10), Row(general=11))`}); err != nil { - t.Fatal(err) - } else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) { - t.Fatalf("unexpected columns: %+v", columns) - } -} - func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) { c := test.MustRunCluster(t, 1, diff --git a/fragment_internal_test.go b/fragment_internal_test.go index f96ecafaa..a5a38042c 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -5110,9 +5110,6 @@ func TestImportValueConcurrent(t *testing.T) { "blueGreenTx because the lack of transactional consistency " + "from Roaring-per-file will create false comparison " + "failures.")) - case lmdbTxn: - t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " + - "lmdb since only a single writer is allowed at once.")) } } diff --git a/lmdb.go b/lmdb.go deleted file mode 100644 index 28075de2d..000000000 --- a/lmdb.go +++ /dev/null @@ -1,1738 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 - -package pilosa - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "log" - "math" - "os" - "path/filepath" - "runtime" - "sort" - "strings" - "sync" - "sync/atomic" - - "github.com/glycerine/lmdb-go/lmdb" - "github.com/pilosa/pilosa/v2/hash" - "github.com/pilosa/pilosa/v2/rbf" - rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg" - "github.com/pilosa/pilosa/v2/roaring" - "github.com/pilosa/pilosa/v2/txkey" - "github.com/pkg/errors" -) - -const TxInitialMmapSize = 4 << 30 // 4GB - -// Linux builds note: -// -// This setting of -DMDB_USE_SYSV_SEM=1 is required in the cgo build -// under Linux to avoid a random deadlock occurance deep inside -// the LMDB C code. -// -// Futher documentation here: https://github.com/bmatsuo/lmdb-go/issues/94 -// -// The effect of the -D define above is to that the LMDB C code then -// uses SysV semaphores and avoids POSIX thread-local semaphores. SysV semaphores -// are stored system-wide. POSIX semaphores are kept on thread-local storage, -// which is apparently problematic. - -// lmdbRegistrar facilitates shutdown -// of all the lmdb databases started under -// tests. Its needed because most tests don't cleanup -// the *Index(es) they create. But we still -// want to shutdown lmdbDB goroutines -// after tests run. -// -// It also allows opening the same path twice to -// result in sharing the same open database handle, and -// thus the same transactional guarantees. -// -type lmdbRegistrar struct { - mu sync.Mutex - mp map[*LMDBWrapper]bool - - path2db map[string]*LMDBWrapper -} - -func (r *lmdbRegistrar) Size() int { - r.mu.Lock() - defer r.mu.Unlock() - nmp := len(r.mp) - npa := len(r.path2db) - if nmp != npa { - panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa)) - } - return nmp -} - -var globalLMDBReg *lmdbRegistrar = newLMDBTestRegistrar() - -var globalNextTxSnLMDB int64 - -func newLMDBTestRegistrar() *lmdbRegistrar { - - return &lmdbRegistrar{ - mp: make(map[*LMDBWrapper]bool), - path2db: make(map[string]*LMDBWrapper), - } -} - -// register each lmdb created under tests, so we -// can clean them up. This is called by openLMDBWrapper() while -// holding the r.mu.Lock, since it needs to atomically -// check the registry and make a new instance only -// if one does not exist for its path, and otherwise -// return the existing instance. -func (r *lmdbRegistrar) unprotectedRegister(w *LMDBWrapper) { - r.mp[w] = true - r.path2db[w.path] = w -} - -// unregister removes w from r -func (r *lmdbRegistrar) unregister(w *LMDBWrapper) { - r.mu.Lock() - delete(r.mp, w) - delete(r.path2db, w.path) - r.mu.Unlock() -} - -func DumpAllLMDB() { - short := true - globalLMDBReg.mu.Lock() - defer globalLMDBReg.mu.Unlock() - for w := range globalLMDBReg.mp { - AlwaysPrintf("this lmdb path='%v' has: \n%v\n", w.path, w.StringifiedLMDBKeys(nil, short)) - } -} - -// lmdbPath is a helper for determining the full directory -// in which the lmdb database will be stored. -func lmdbPath(path string) string { - if !strings.HasSuffix(path, "-lmdb@") { - return path + "-lmdb@" - } - return path -} - -// openLMDBDB opens the database in the bpath directoy -// without deleting any prior content. Any LMDBDB -// database directory will have the "-lmdb" suffix. -// -// openLMDBDB will check the registry and make a new instance only -// if one does not exist for its bpath. Otherwise it returns -// the existing instance. This insures only one lmdbDB -// per bpath in this pilosa node. - -func (r *lmdbRegistrar) OpenDBWrapper(path0 string, doAllocZero bool, rbfcfg *rbfcfg.Config) (DBWrapper, error) { - path := lmdbPath(path0) - - r.mu.Lock() - defer r.mu.Unlock() - w, ok := r.path2db[path] - if ok { - // creates the effect of having only one lmdb open per pilosa node. - return w, nil - } - // otherwise, make a new lmdb and store it in globalLMDBReg - - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - const MaxReaders = 126 // default is 126 - env, err := lmdb.NewEnvMaxReaders(MaxReaders) - panicOn(err) - - err = env.SetMaxDBs(1) - panicOn(err) - //err = env.SetMapSize(256 << 30) // 256GB - err = env.SetMapSize(TxInitialMmapSize) - panicOn(err) - - panicOn(os.MkdirAll(filepath.Dir(path), 0755)) - - //flags := uint(lmdb.NoReadahead | lmdb.NoSubdir) - //flags := uint(lmdb.NoSubdir) // no difference without the No.Readahead on ./query. - // NoReadahead should be better for random workloads or those bigger than memory, - // as it avoids loading extra pages which then evict pages you are using. - //flags := uint(lmdb.NoReadahead) - flags := uint(0) - - // unsafe, but get upper bound on performance. - // WriteMap = C.MDB_WRITEMAP // Use a writable memory map. - // NoMetaSync = C.MDB_NOMETASYNC // Don't fsync metapage after commit. - // NoSync = C.MDB_NOSYNC // Don't fsync after commit. - // flags = flags | lmdb.WriteMap | lmdb.NoMetaSync | lmdb.NoSync // about the same speed - // flags = flags | lmdb.NoMetaSync | lmdb.NoSync // slows things down - - //flags = flags | lmdb.WriteMap // seems faster than without: or maybe not. not sure. - // kRemove N= 710401 avg/op: 7.714µs sd: 27.83µs total: 5.480656859s - // kAdd N= 722835 avg/op: 9.096µs sd: 105.787µs total: 6.575497725s - - // ACI not ACID at the moment; no durability - flags = flags | - lmdb.NoMemInit | // Disable LMDB memory initialization - - // Note that lmdb.WriteMap requests a big, writable, memory map. - // On my darwin/OSX laptop with 16GB ram, for instance, we - // can have difficulty obtaining this, resulting in - // panic: mdb_env_open: no space left on device - lmdb.WriteMap // Use a writable memory map. - - if rbfcfg == nil || !rbfcfg.FsyncEnabled { - // default for lmdb: fsync off. - // unsafe, but get upper bound on performance. - flags = flags | - // default ACI (not Durable) transactions; 300% faster write speed results. - lmdb.NoMetaSync | // Don't fsync metapage after commit. - lmdb.NoSync | // Don't fsync after commit. - lmdb.MapAsync // Flush asynchronously when using the WriteMap flag. - } - - if !DirExists(path) { - panicOn(os.MkdirAll(path, 0755)) - } - err = env.Open(path, flags, 0644) - if err != nil { - AlwaysPrintf("error env.Open(path='%v'): '%v'; on gid = '%v'", path, err, curGID()) - } - panicOn(err) - - // In any real application it is important to check for readers that were - // never closed by their owning process, and for which the owning process - // has exited. See the documentation on transactions for more information. - staleReaders, err := env.ReaderCheck() - panicOn(err) - if staleReaders > 0 { - log.Printf("cleared %d reader slots from dead processes", staleReaders) - } - - // Open a database handle that will be used for the entire lifetime of this - // application. Because the database may not have existed before, and the - // database may need to be created, we need to get the database handle in - // an update transacation. - var dbi lmdb.DBI - name := filepath.Base(path) - err = env.Update(func(txn *lmdb.Txn) (err error) { - dbi, err = txn.CreateDBI(name) - return err - }) - panicOn(err) - - w = &LMDBWrapper{ - name: name, - env: env, - reg: r, - path: path, - dbi: dbi, - doAllocZero: doAllocZero, - openTx: make(map[*LMDBTx]bool), - - DeleteEmptyContainer: true, - } - r.unprotectedRegister(w) - - return w, nil -} - -func (w *LMDBWrapper) Path() string { - return w.path -} - -func (w *LMDBWrapper) HasData() (has bool, err error) { - - tx, err := w.NewTx(!writable, "", Txo{}) - if err != nil { - return false, errors.Wrap(err, "HasData NewTx") - } - defer tx.Rollback() - - bi := NewLMDBIterator(tx.(*LMDBTx), nil) - defer bi.Close() - - for bi.Next() { - return true, nil - } - return false, nil -} - -func (w *LMDBWrapper) CleanupTx(tx Tx) { - // inlined into Rollback and Commit, so this is a no-op, just here to satisfy the interface. -} - -func (tx *LMDBTx) IsDone() (done bool) { - return atomic.LoadInt64(&tx.unlocked) == 1 -} - -func (w *LMDBWrapper) OpenListString() (r string) { - - list := w.listopen() - if len(list) == 0 { - return "" - } - for i, ltx := range list { - if ltx.o.Write { - r += fmt.Sprintf("[%v]write: _sn_ %v %v, \n", i, ltx.sn, ltx.o) - } else { - r += fmt.Sprintf("[%v]read : _sn_ %v %v, \n", i, ltx.sn, ltx.o) - } - } - return -} - -func (w *LMDBWrapper) listopen() (slc []*LMDBTx) { - w.muDb.Lock() - for v := range w.openTx { - slc = append(slc, v) - } - w.muDb.Unlock() - return -} - -func (w *LMDBWrapper) OpenSnList() (slc []int64) { - w.muDb.Lock() - for v := range w.openTx { - slc = append(slc, v.sn) - } - w.muDb.Unlock() - return -} - -var ErrShutdown = fmt.Errorf("shutting down") - -// DeleteIndex deletes all the containers associated with -// the named index from the lmdb database. -func (w *LMDBWrapper) DeleteIndex(indexName string) error { - - // We use the apostrophie rune `'` to locate the end of the - // index name in the key prefix, so we cannot allow indexNames - // themselves to contain apostrophies. - if strings.Contains(indexName, "/") { - return fmt.Errorf("error: bad indexName `%v` in LMDBWrapper.DeleteIndex() call: indexName cannot contain '/'.", indexName) - } - prefix := txkey.IndexOnlyPrefix(indexName) - return w.DeletePrefix(prefix) -} - -// statically confirm that LMDBTx satisfies the Tx interface. -var _ Tx = (*LMDBTx)(nil) - -// LMDBWrapper provides the NewTx() method. -type LMDBWrapper struct { - env *lmdb.Env - - muDb sync.Mutex - - path string - name string - dbi lmdb.DBI - - // track our registrar for Close / goro leak reporting purposes. - reg *lmdbRegistrar - - // make LMDBWrapper.Close() idempotent, avoiding panic on double Close() - closed bool - - // doAllocZero sets the corresponding flag on all new LMDBTx. - // When doAllocZero is true, we zero out any data from lmdb - // after transcation commit and rollback. This simulates - // what would happen if we were to use the mmap-ed data - // from lmdb directly. Currently we copy by default for - // safety because otherwise TestAPI_ImportColumnAttrs sees - // corrupted data. - doAllocZero bool - - DeleteEmptyContainer bool - - openTx map[*LMDBTx]bool -} - -func (w *LMDBWrapper) SetHolder(h *Holder) { - // don't need it at the moment - //w.h = h -} - -// NewTxWRITE lets us see in the callstack dumps where the WRITE tx are. -// Can't have more than one active write per database, so the -// 2nd one will block until the first finishes. -func (w *LMDBWrapper) NewTxWRITE() (*lmdb.Txn, error) { - lmdbTxn, err := w.env.BeginTxn(nil, 0) - if err != nil { - if w.env == nil || w.IsClosed() { - return nil, fmt.Errorf("cannot call NewTxWRITE() on closed LMDB database: '%v'", err) - } - return nil, err - } - return lmdbTxn, nil -} - -// NewTxREAD lets us see in the callstack dumps where the READ tx are. -func (w *LMDBWrapper) NewTxREAD() (*lmdb.Txn, error) { - lmdbTxn, err := w.env.BeginTxn(nil, lmdb.Readonly) - if err != nil { - if w.env == nil || w.IsClosed() { - return nil, fmt.Errorf("cannot call NewTxREAD() on closed LMDB database: '%v'", err) - } - return nil, err - } - return lmdbTxn, nil -} - -// NewTx produces LMDB based ACID or ACI transactions. If -// the transaction will modify data, then the write flag must be true. -// Read-only queries should set write to false, to allow more concurrency. -// Methods on a LMDBTx are thread-safe, and can be called from -// different goroutines. -// -// initialIndexName is optional. It is set by the TxFactory from the Txo -// options provided at the Tx creation point. It allows us to recognize -// and isolate cross-index queries more quickly. It can always be empty "" -// but when set is highly useful for debugging. It has no impact -// on transaction behavior. -// -func (w *LMDBWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { - - sn := atomic.AddInt64(&globalNextTxSnLMDB, 1) - - //vv("lmdb new tx _sn_ %v; openTx='%v', stack \n%v", sn, w.OpenListString(), stack()) - //vv("lmdb new (write=%v, shard=%v) tx _sn_ %v; openTx='%v'", write, o.Shard, sn, w.OpenListString()) - - runtime.LockOSThread() - - var lmdbTxn *lmdb.Txn - if write { - // see the WRITE tx on the callstack. - lmdbTxn, err = w.NewTxWRITE() - if err != nil { - return nil, err - } - } else { - // see the READ tx on the callstack. - lmdbTxn, err = w.NewTxREAD() - if err != nil { - return nil, err - } - } - - lmdbTxn.RawRead = true - - ltx := &LMDBTx{ - sn: sn, - write: write, - tx: lmdbTxn, - dbi: w.dbi, - Db: w, - frag: o.Fragment, - doAllocZero: w.doAllocZero, - initialIndexName: initialIndexName, - DeleteEmptyContainer: w.DeleteEmptyContainer, - o: o, - gid: curGID(), - } - tx = ltx - - if isDebugRun { - w.muDb.Lock() - w.openTx[ltx] = true - w.muDb.Unlock() - } - return -} - -// Close shuts down the LMDB database. -func (w *LMDBWrapper) Close() (err error) { - w.muDb.Lock() - defer w.muDb.Unlock() - if !w.closed { - if isDebugRun { - // complain if there are still Tx in flight, b/c otherwise we will see - // the somewhat mysterious 'panic: should not be in ReadSlot.free() with slot still owned by gid=107043; refCount=1' - if len(w.openTx) > 0 { - AlwaysPrintf("error: cannot close LMDBWrapper with Tx still in flight.") - return - } - } - w.reg.unregister(w) - w.closed = true - w.env.CloseDBI(w.dbi) - w.env.Close() - w.env = nil - } - return nil -} - -func (w *LMDBWrapper) IsClosed() (closed bool) { - w.muDb.Lock() - closed = w.closed - w.muDb.Unlock() - return -} - -// LMDBTx wraps a lmdb.Txn and provides the Tx interface -// method implementations. -// The methods on LMDBTx are thread-safe, and can be called -// from different goroutines. -type LMDBTx struct { - - // mu serializes lmdb operations on this single txn instance. - mu sync.Mutex - sn int64 // serial number - - write bool - dbi lmdb.DBI - Db *LMDBWrapper - tx *lmdb.Txn - frag *fragment - - opcount int - - //initloc string // stack trace of where we were initially created. - - doAllocZero bool - - initialIndexName string - - DeleteEmptyContainer bool - - unlocked int64 // runtime.UnlockOSThread has been done if > 0 - - o Txo - - // NewTx, write operations, Commit and/or Rollback must all take place on - // the same gid and it must the runtime.LockOSThreaded first. Verify - // that we are using the right goroutine in a debug build using the - // gid, stored here, used for NewTx(). - gid uint64 -} - -// sanity check that database is open. -func (tx *LMDBTx) sanity() { - if tx.Db.IsClosed() { - panic("cannot operate on closed LMDB") - } -} - -// debugOnlyGidcheck is expected to be sort of slow. So once we are sure -// of correctness, turn it off. -// func (tx *LMDBTx) debugOnlyGidcheck() { -// -// // only applies to write txn. Reads should be able to share the txn. -// if tx.o.Write { -// cur := curGID() -// if cur != tx.gid { -// panic(fmt.Sprintf("must use write LMDBTx from same gid that created it! creator gid=%v, but user gid now =%v", tx.gid, cur)) -// } -// } -// -// } - -func (tx *LMDBTx) Group() *TxGroup { - return tx.o.Group -} - -func (tx *LMDBTx) Type() string { - return LmdbTxn -} - -func (tx *LMDBTx) UseRowCache() bool { - return rbf.EnableRowCache() -} - -// Pointer gives us a memory address for the underlying transaction for debugging. -// It is public because we use it in roaring to report invalid container memory access -// outside of a transaction. -func (tx *LMDBTx) Pointer() string { - return fmt.Sprintf("%p", tx) -} - -const isDebugRun = false - -// Rollback rolls back the transaction. -func (tx *LMDBTx) Rollback() { - notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) - if !notDone { - return - } - //vv("lmdb rollback tx _sn_ %v; stack \n%v", tx.sn) // , stack()) - if isDebugRun { - tx.sanity() - tx.Db.muDb.Lock() - delete(tx.Db.openTx, tx) - tx.Db.muDb.Unlock() - } - - tx.mu.Lock() - defer tx.mu.Unlock() - - //tx.debugOnlyGidcheck() - tx.tx.Abort() // must hold tx.mu mutex lock - - // use CAS above instead of testing a bool unlocked. - runtime.UnlockOSThread() - tx.o.dbs.Cleanup(tx) -} - -// Commit commits the transaction to permanent storage. -// Commits can handle up to 100k updates to fragments -// at once, but not more. This is a LMDBDB imposed limit. -func (tx *LMDBTx) Commit() error { - notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) - if !notDone { - return nil - } - //vv("lmdb commit tx _sn_ %v; stack \n%v", tx.sn, stack()) - if isDebugRun { - tx.sanity() - tx.Db.muDb.Lock() - delete(tx.Db.openTx, tx) - tx.Db.muDb.Unlock() - } - tx.mu.Lock() - defer tx.mu.Unlock() - - //tx.debugOnlyGidcheck() - err := tx.tx.Commit() // must hold tx.mu mutex lock - panicOn(err) - - // replace the if !tx.unlocked with the CAS on tx.unlocked above. - runtime.UnlockOSThread() - tx.o.dbs.Cleanup(tx) - return err -} - -// Readonly returns true iff the LMDBTx is read-only. -func (tx *LMDBTx) Readonly() bool { - return !tx.write -} - -// RoaringBitmap returns the roaring.Bitmap for all bits in the fragment. -func (tx *LMDBTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - - return tx.OffsetRange(index, field, view, shard, 0, 0, LeftShifted16MaxContainerKey) -} - -// Container returns the requested roaring.Container, selected by fragment and ckey -func (tx *LMDBTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) { - - // values returned from Get() are only valid while the transaction - // is open. If you need to use a value outside of the transaction then - // you must use copy() to copy it to another byte slice. - // BUT here we are already inside the Txn. - - bkey := txkey.Key(index, field, view, shard, ckey) - tx.mu.Lock() - - //tx.debugOnlyGidcheck() - - v, err := tx.tx.Get(tx.dbi, bkey) - tx.mu.Unlock() - - if lmdb.IsNotFound(err) { - // Seems crazy, but we, for now at least, - // match what RoaringTx does by returning nil, nil. - return nil, nil - } else { - if err != nil { - panicOn(err) - return nil, nil - } - } - n := len(v) - if n > 0 { - c = tx.toContainer(v[n-1], v[0:(n-1)]) - } - return -} - -// PutContainer stores rc under the specified fragment and container ckey. -func (tx *LMDBTx) PutContainer(index, field, view string, shard uint64, ckey uint64, rc *roaring.Container) error { - - bkey := txkey.Key(index, field, view, shard, ckey) - var by []byte - - ct := roaring.ContainerType(rc) - - switch ct { - case roaring.ContainerArray: - by = fromArray16(roaring.AsArray(rc)) - case roaring.ContainerBitmap: - by = fromArray64(roaring.AsBitmap(rc)) - case roaring.ContainerRun: - by = fromInterval16(roaring.AsRuns(rc)) - case roaring.ContainerNil: - panic("wat? nil container is unexpected, no?!?") - default: - panic(fmt.Sprintf("unknown container type: %v", ct)) - } - tx.mu.Lock() - //tx.debugOnlyGidcheck() - - err := tx.tx.Put(tx.dbi, bkey, append(by, ct), 0) // TODO: this might make a copy; can meta byte be stored elsewhere? - tx.mu.Unlock() - - // TODO(jea): need to handle? - // lmdb.TxnFull - // lmdb.CursorFull - // lmdb.PageFull - return err -} - -// RemoveContainer deletes the container specified by the shard and container key ckey -func (tx *LMDBTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error { - bkey := txkey.Key(index, field, view, shard, ckey) - tx.mu.Lock() - //tx.debugOnlyGidcheck() - - err := tx.tx.Del(tx.dbi, bkey, nil) - tx.mu.Unlock() - if lmdb.IsNotFound(err) { - return nil - } - return err -} - -// Add sets all the a bits hot in the specified fragment. -func (tx *LMDBTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) { - return tx.addOrRemove(index, field, view, shard, batched, false, a...) -} - -// Remove clears all the specified a bits in the chosen fragment. -func (tx *LMDBTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - const batched = false - const remove = true - return tx.addOrRemove(index, field, view, shard, batched, remove, a...) -} - -func (tx *LMDBTx) addOrRemove(index, field, view string, shard uint64, batched, remove bool, a ...uint64) (changeCount int, err error) { - // pure hack to match RoaringTx - defer func() { - if !remove && !batched { - if changeCount > 0 { - changeCount = 1 - } - } - }() - - if len(a) == 0 { - return 0, nil - } - - // have to sort, b/c input is not always sorted. - sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) - - var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. - var rc *roaring.Container - var hi uint64 - var lo uint16 - - for i, v := range a { - - hi, lo = highbits(v), lowbits(v) - if hi != lastHi { - // either first time through, or changed to a different container. - // do we need put the last updated container now? - if i > 0 { - // not first time through, write what we got. - if remove && (rc == nil || rc.N() == 0) { - err = tx.RemoveContainer(index, field, view, shard, lastHi) - panicOn(err) - } else { - err = tx.PutContainer(index, field, view, shard, lastHi, rc) - panicOn(err) - } - } - // get the next container - rc, err = tx.Container(index, field, view, shard, hi) - panicOn(err) - } // else same container, keep adding bits to rct. - chng := false - // rc can be nil before, and nil after, in both Remove/Add below. - // The roaring container add() and remove() methods handle this. - if remove { - rc, chng = rc.Remove(lo) - } else { - rc, chng = rc.Add(lo) - } - if chng { - changeCount++ - } - lastHi = hi - } - // write the last updates. - if remove { - if rc == nil || rc.N() == 0 { - err = tx.RemoveContainer(index, field, view, shard, hi) - panicOn(err) - } else { - err = tx.PutContainer(index, field, view, shard, hi, rc) - panicOn(err) - } - } else { - if rc == nil || rc.N() == 0 { - panic("there should be no way to have an empty bitmap AFTER an Add() operation") - } - err = tx.PutContainer(index, field, view, shard, hi, rc) - panicOn(err) - } - return -} - -// Contains returns exists true iff the bit chosen by key is -// hot (set to 1) in specified fragment. -func (tx *LMDBTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { - - lo, hi := lowbits(key), highbits(key) - bkey := txkey.Key(index, field, view, shard, hi) - tx.mu.Lock() - var v []byte - - //tx.debugOnlyGidcheck() - - v, err = tx.tx.Get(tx.dbi, bkey) - tx.mu.Unlock() - if lmdb.IsNotFound(err) { - return false, nil - } - if err != nil { - return false, err - } - n := len(v) - if n > 0 { - c := tx.toContainer(v[n-1], v[0:(n-1)]) - exists = c.Contains(lo) - } - return exists, err -} - -func (tx *LMDBTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { - - prefix := txkey.AllShardPrefix(index, field, view) - - bi := NewLMDBIterator(tx, prefix) - defer bi.Close() - - lastShard := uint64(0) - firstDone := false - for bi.Next() { - shard := txkey.ShardFromKey(bi.lastKey) - if firstDone { - if shard != lastShard { - sliceOfShards = append(sliceOfShards, shard) - } - lastShard = shard - } else { - // first time - lastShard = shard - firstDone = true - sliceOfShards = append(sliceOfShards, shard) - } - - } - return -} - -// key is the container key for the first roaring Container -// roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will -// return the first container at or after key. found will be true if a -// container is found at key. -// -// LMDBTx notes: We auto-stop at the end of this shard, not going beyond. -func (tx *LMDBTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { - - // needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000" - needle := txkey.Key(index, field, view, shard, firstRoaringContainerKey) - - // prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@" - prefix := txkey.Prefix(index, field, view, shard) - - bi := NewLMDBIterator(tx, prefix) - ok := bi.Seek(needle) - if !ok { - return bi, false, nil - } - - // have to compare b/c lmdb might give us valid iterator - // that is past our needle if needle isn't present. - return bi, bytes.Equal(bi.lastKey, needle), nil -} - -// LMDBIterator is the iterator returned from a LMDBTx.ContainerIterator() call. -// It implements the roaring.ContainerIterator interface. -type LMDBIterator struct { - tx *LMDBTx - cur *lmdb.Cursor - dbi lmdb.DBI - - prefix []byte - seekto []byte - - // seen counts how many Next() calls we have seen. - // It is used to match roaring.ContainerIterator semantics. - // Also useful for testing. - seen int - - lastKey []byte - lastVal []byte // *roaring.Container - lastOK bool - lastConsumed bool -} - -// NewLMDBIterator creates an iterator on tx that will -// only return rbf that start with prefix. -func NewLMDBIterator(tx *LMDBTx, prefix []byte) (bi *LMDBIterator) { - - tx.mu.Lock() - //tx.debugOnlyGidcheck() - - cur, err := tx.tx.OpenCursor(tx.dbi) - tx.mu.Unlock() - - panicOn(err) - - bi = &LMDBIterator{ - dbi: tx.dbi, - tx: tx, - cur: cur, - prefix: prefix, - } - return -} - -// Close tells the database and transaction that the user is done -// with the iterator. -func (bi *LMDBIterator) Close() { - bi.tx.mu.Lock() - //bi.tx.debugOnlyGidcheck() - - bi.cur.Close() - bi.tx.mu.Unlock() -} - -// Valid returns false if there are no more values in the iterator's range. -func (bi *LMDBIterator) Valid() bool { - return bi.lastOK -} - -// Seek allows the iterator to start at needle instead of the global begining. -func (bi *LMDBIterator) Seek(needle []byte) (ok bool) { - bi.tx.mu.Lock() - defer bi.tx.mu.Unlock() - - bi.seen++ // if ommited, red TestLMDB_ContainerIterator_empty_iteration_loop() in lmdb_test.go. - - //bi.tx.debugOnlyGidcheck() - - var k, v []byte - var err error - getflag := uint(lmdb.SetRange) - if len(needle) == 0 { - k, v, err = bi.cur.Get(oneByteSliceOfZero, nil, getflag) - } else { - k, v, err = bi.cur.Get(needle, nil, getflag) - } - - if lmdb.IsNotFound(err) { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - if len(bi.prefix) > 0 { - ok = bytes.HasPrefix(k, bi.prefix) - if !ok { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - } - if len(k) == 0 { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - - bi.lastKey = k - bi.lastVal = v - bi.lastOK = true - bi.lastConsumed = false - - return true -} - -func (bi *LMDBIterator) ValidForPrefix(prefix []byte) bool { - if !bi.lastOK { - return false - } - if len(bi.prefix) == 0 { - return true - } - return bytes.HasPrefix(bi.lastKey, bi.prefix) -} - -func (bi *LMDBIterator) String() (r string) { - return fmt.Sprintf("LMDBIterator{prefix: '%v', seekto: '%v', seen:%v, lastKey:'%v', lastOK:%v, lastConsumed:%v}", string(bi.prefix), string(bi.seekto), bi.seen, string(bi.lastKey), bi.lastOK, bi.lastConsumed) -} - -var oneByteSliceOfZero = []byte{0} - -// Next advances the iterator. -func (bi *LMDBIterator) Next() (ok bool) { - if bi.lastOK && !bi.lastConsumed { - bi.seen++ - bi.lastConsumed = true - if len(bi.lastVal) == 0 { - panic("bi.lastVal should not have len 0 if lastOK true") - } - return true - } - - getflag := uint(lmdb.Next) - prefix := bi.prefix - - if bi.seen == 0 { - if len(bi.prefix) > 0 { - getflag = lmdb.SetRange - } - } else { - prefix = nil - } - - bi.seen++ -skipEmpty: - var k, v []byte - var err error - - bi.tx.mu.Lock() - if getflag == lmdb.SetRange && len(prefix) == 0 { - // don't do nil as key on setrange, will panic - // b/c keys in LMDB must be at least one byte long. - // http://www.lmdb.tech/doc/group__mdb.html#structMDB__val - // "Key sizes must be between 1 and mdb_env_get_maxkeysize() inclusive." - // But if getflag == lmdb.Next, key can be nil. - k, v, err = bi.cur.Get(oneByteSliceOfZero, nil, getflag) - } else { - k, v, err = bi.cur.Get(prefix, nil, getflag) - } - bi.tx.mu.Unlock() - - //bi.tx.debugOnlyGidcheck() - - if lmdb.IsNotFound(err) { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - if len(bi.prefix) > 0 { - ok = bytes.HasPrefix(k, bi.prefix) - if !ok { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - } - bi.lastKey = k - bi.lastVal = v - if len(v) == 0 { - // actually under !tx.DeleteEmptyContainer, we can have empty containers! - goto skipEmpty - } - bi.lastOK = true - bi.lastConsumed = true - - return true -} - -// Value retrieves what is pointed at currently by the iterator. -func (bi *LMDBIterator) Value() (containerKey uint64, c *roaring.Container) { - if !bi.lastOK { - panic("bi.cur not valid") - } - containerKey = txkey.KeyExtractContainerKey(bi.lastKey) - - v := bi.lastVal - n := len(v) - if n > 0 { - c = bi.tx.toContainer(v[n-1], v[0:(n-1)]) - } else { - panic("v should not be empty!") - } - return -} - -// lmdbFinder implements roaring.IteratorFinder. -// It is used by LMDBTx.ForEach() -type lmdbFinder struct { - tx *LMDBTx - index string - field string - view string - shard uint64 - needClose []Closer -} - -// FindIterator lets lmdbFinder implement the roaring.FindIterator interface. -func (bf *lmdbFinder) FindIterator(seek uint64) (roaring.ContainerIterator, bool) { - a, found, err := bf.tx.ContainerIterator(bf.index, bf.field, bf.view, bf.shard, seek) - panicOn(err) - bf.needClose = append(bf.needClose, a) - return a, found -} - -// Close closes all bf.needClose listed Closers. -func (bf *lmdbFinder) Close() { - for _, i := range bf.needClose { - i.Close() - } -} - -// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE -// the transaction Commits or Rollsback. -func (tx *LMDBTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { - - bf := &lmdbFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} - itr := roaring.NewIterator(bf) - return itr -} - -// ForEach applies fn to each bitmap in the fragment. -func (tx *LMDBTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { - - itr := tx.NewTxIterator(index, field, view, shard) - defer itr.Close() - - // Seek can create many container iterators, thus bf.Close() needClose list. - itr.Seek(0) - // v is the bit we are operating on. - for v, eof := itr.Next(); !eof; v, eof = itr.Next() { - if err := fn(v); err != nil { - return err - } - } - return nil -} - -// ForEachRange applies fn on the selected range of bits on the chosen fragment. -func (tx *LMDBTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { - - itr := tx.NewTxIterator(index, field, view, shard) - defer itr.Close() - - itr.Seek(start) - - // v is the bit we are operating on. - for v, eof := itr.Next(); !eof && v < end; v, eof = itr.Next() { - if err := fn(v); err != nil { - return err - } - } - return nil -} - -// Count operates on the full bitmap level, so it sums over all the containers -// in the bitmap. -func (tx *LMDBTx) Count(index, field, view string, shard uint64) (uint64, error) { - - a, found, err := tx.ContainerIterator(index, field, view, shard, 0) - panicOn(err) - defer a.Close() - if !found { - return 0, nil - } - result := int32(0) - for a.Next() { - ckey, cont := a.Value() - _ = ckey - result += cont.N() - } - return uint64(result), nil -} - -// Max is the maximum bit-value in your bitmap. -// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does. -func (tx *LMDBTx) Max(index, field, view string, shard uint64) (uint64, error) { - - prefix := txkey.Prefix(index, field, view, shard) - seekto := txkey.Prefix(index, field, view, shard+1) - - //tx.debugOnlyGidcheck() - - cur, err := tx.tx.OpenCursor(tx.dbi) - panicOn(err) - defer cur.Close() - - var k, v []byte - if len(seekto) == 0 { - _, _, err = cur.Get(oneByteSliceOfZero, nil, lmdb.SetRange) - } else { - _, _, err = cur.Get(seekto, nil, lmdb.SetRange) - } - if lmdb.IsNotFound(err) { - // we have nothing >= seekto, but we might have stuff before it, and we'll wrap backwards. - k, v, err = cur.Get(nil, nil, lmdb.Prev) - if lmdb.IsNotFound(err) { - // empty database - return 0, nil - } - } else { - // we found something >= seekto, so backup by 1. - k, v, err = cur.Get(nil, nil, lmdb.Prev) - if lmdb.IsNotFound(err) { - // nothing before seekto - return 0, nil - } - } - - // have something, are we in [prefix, seekto) ? - cmp := bytes.Compare(k, prefix) - if cmp >= 0 { - // good, got max in k, v - } else { - return 0, nil // nothing in [prefix, seekto). - } - - hb := txkey.KeyExtractContainerKey(k) - n := len(v) - if n == 0 { - return 0, nil - } - rc := tx.toContainer(v[n-1], v[0:(n-1)]) - - lb := rc.Max() - return hb<<16 | uint64(lb), nil -} - -// Min returns the smallest bit set in the fragment. If no bit is hot, -// the second return argument is false. -func (tx *LMDBTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { - - // Seek can create many container iterators, thus the bf.Close() needClose list. - bf := &lmdbFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} - defer bf.Close() - itr := roaring.NewIterator(bf) - - itr.Seek(0) - - // v is the bit we are operating on. - v, eof := itr.Next() - if eof { - return 0, false, nil - } - return v, true, nil -} - -// UnionInPlace unions all the others Bitmaps into a new Bitmap, and then writes it to the -// specified fragment. -func (tx *LMDBTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - panicOn(err) - - rbm.UnionInPlace(others...) - // iterate over the containers that changed within rbm, and write them back to disk. - - it, found := rbm.Containers.Iterator(0) - _ = found // don't care about the value of found, because first containerKey might be > 0 - - for it.Next() { - containerKey, rc := it.Value() - - // TODO: only write the changed ones back, as optimization? - // Compare to ImportRoaringBits. - err := tx.PutContainer(index, field, view, shard, containerKey, rc) - panicOn(err) - } - return nil -} - -// CountRange returns the count of hot bits in the start, end range on the fragment. -// roaring.countRange counts the number of bits set between [start, end). -func (tx *LMDBTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { - - if start >= end { - return 0, nil - } - - skey := highbits(start) - ekey := highbits(end) - - citer, found, err := tx.ContainerIterator(index, field, view, shard, skey) - _ = found - panicOn(err) - - defer citer.Close() - - // If range is entirely in one container then just count that range. - if skey == ekey { - citer.Next() - _, c := citer.Value() - return uint64(c.CountRange(int32(lowbits(start)), int32(lowbits(end)))), nil - } - - for citer.Next() { - k, c := citer.Value() - if k < skey { - citer.Close() - panic(fmt.Sprintf("should be impossible for k(%v) to be less than skey(%v). tx p=%p", k, skey, tx)) - } - - // k > ekey handles the case when start > end and where start and end - // are in different containers. Same container case is already handled above. - if k > ekey { - break - } - if k == skey { - n += uint64(c.CountRange(int32(lowbits(start)), roaring.MaxContainerVal+1)) - continue - } - if k < ekey { - n += uint64(c.N()) - continue - } - if k == ekey { - n += uint64(c.CountRange(0, int32(lowbits(end)))) - break - } - } - - return n, nil -} - -// OffsetRange creates a new roaring.Bitmap to return in other. For all the -// hot bits in [start, endx) of the chosen fragment, it stores -// them into other but with offset added to their bit position. -// The primary client is doing this, using ShardWidth, already; see -// fragment.rowFromStorage() in fragment.go. For example: -// -// data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard, -// -// f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) -// ^ offset ^ start ^ endx -// -// The start and endx arguments are container keys that have been shifted left by 16 bits; -// their highbits() will be taken to determine the actual container keys. This -// is done to conform to the roaring.OffsetRange() argument convention. -// -func (tx *LMDBTx) OffsetRange(index, field, view string, shard, offset, start, endx uint64) (other *roaring.Bitmap, err error) { - //vv("top of LMDBTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v)", index, field, view, int(shard), int(offset), int(start), int(endx)) - //defer func() { - //vv("returning from LMDBTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) other returning is: '%#v' stack=\n%v", index, field, view, int(shard), int(offset), int(start), int(endx), asInts(other.Slice()), stack()) - //}() - - // roaring does these three checks in its OffsetRange - if lowbits(offset) != 0 { - panic("offset must not contain low bits") - } - if lowbits(start) != 0 { - panic("range start must not contain low bits") - } - if lowbits(endx) != 0 { - panic("range end must not contain low bits") - } - - other = roaring.NewSliceBitmap() - off := highbits(offset) - hi0, hi1 := highbits(start), highbits(endx) - - needle := txkey.Key(index, field, view, shard, hi0) - prefix := txkey.Prefix(index, field, view, shard) - - it := NewLMDBIterator(tx, prefix) - defer it.Close() - it.Seek(needle) - for ; it.ValidForPrefix(prefix); it.Next() { - bkey := it.lastKey - k := txkey.KeyExtractContainerKey(bkey) - - // >= hi1 is correct b/c endx cannot have any lowbits set. - if uint64(k) >= hi1 { - break - } - destCkey := off + (k - hi0) - - v := it.lastVal - n := len(v) - if n == 0 { - continue - } - c := tx.toContainer(v[n-1], v[0:(n-1)]) - other.Containers.Put(destCkey, c.Freeze()) - } - return other, nil -} - -// IncrementOpN increments the tx opcount by changedN -func (tx *LMDBTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - tx.opcount += changedN -} - -// ImportRoaringBits handles deletes by setting clear=true. -// rowSet[rowID] returns the number of bit changed on that rowID. -func (tx *LMDBTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { - - n := itr.Len() - if n == 0 { - return - } - rowSet = make(map[uint64]int) - - var currRow uint64 - - var oldC *roaring.Container - for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { - if rowSize != 0 { - currRow = itrKey / rowSize - } - nsynth := int(synthC.N()) - if nsynth == 0 { - continue - } - // INVAR: nsynth > 0 - - oldC, err = tx.Container(index, field, view, shard, itrKey) - panicOn(err) - if err != nil { - return - } - - if oldC == nil || oldC.N() == 0 { - // no container at the itrKey in lmdb (or all zero container). - if clear { - // changed of 0 and empty rowSet is perfect, no need to change the defaults. - continue - } else { - - changed += nsynth - rowSet[currRow] += nsynth - - err = tx.PutContainer(index, field, view, shard, itrKey, synthC) - if err != nil { - return - } - continue - } - } - - if clear { - existN := oldC.N() // number of bits set in the old container - newC := oldC.Difference(synthC) - - // update rowSet and changes - if newC.N() == existN { - // INVAR: do changed need adjusting? nope. same bit count, - // so no change could have happened. - continue - } else { - changes := int(existN - newC.N()) - changed += changes - rowSet[currRow] -= changes - - if tx.DeleteEmptyContainer && newC.N() == 0 { - err = tx.RemoveContainer(index, field, view, shard, itrKey) - if err != nil { - return - } - continue - } - err = tx.PutContainer(index, field, view, shard, itrKey, newC) - if err != nil { - return - } - continue - } - } else { - // setting bits - - existN := oldC.N() - if existN == roaring.MaxContainerVal+1 { - // completely full container already, set will do nothing. so changed of 0 default is perfect. - continue - } - if existN == 0 { - // can nsynth be zero? No, because of the continue/invariant above where nsynth > 0 - changed += nsynth - rowSet[currRow] += nsynth - err = tx.PutContainer(index, field, view, shard, itrKey, synthC) - if err != nil { - return - } - continue - } - - newC := roaring.Union(oldC, synthC) // UnionInPlace was giving us crashes on overly large containers. - - if roaring.ContainerType(newC) == roaring.ContainerBitmap { - newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it. - } - if newC.N() != existN { - changes := int(newC.N() - existN) - changed += changes - rowSet[currRow] += changes - - err = tx.PutContainer(index, field, view, shard, itrKey, newC) - if err != nil { - panicOn(err) - return - } - continue - } - } - } - return -} - -func (tx *LMDBTx) toContainer(typ byte, v []byte) (r *roaring.Container) { - - //tx.debugOnlyGidcheck() - - if len(v) == 0 { - return nil - } - - var w []byte - useRowCache := tx.UseRowCache() - if tx.doAllocZero || useRowCache { - // Do electric fence-inspired bad-memory read detection. - // - // The v []byte lives in LMDBDB's memory-mapped vlog-file, - // and LMDB will recycle it after tx ends with rollback or commit. - // - // Problem is, at least some operations were not respecting transaction boundaries. - // This technique helped us find them. The rowCache was an example. - // - // See the global const DetectMemAccessPastTx - // at the top of txfactory.go to activate/deactivate this. - // - // Seebs suggested this nice variation: we could use individual mmaps for these - // copies, which would be unusable in production, but workable for testing, and then unmap them, - // which would get us probable segfaults on future accesses to them. - // - // The go runtime also has an -efence flag which may be similarly useful if really pressed. - // - w = make([]byte, len(v)) - copy(w, v) - } else { - w = v - } - return ToContainer(typ, w) -} - -func ToContainer(typ byte, w []byte) (c *roaring.Container) { - switch typ { - case roaring.ContainerArray: - c = roaring.NewContainerArray(toArray16(w)) - case roaring.ContainerBitmap: - c = roaring.NewContainerBitmap(-1, toArray64(w)) - case roaring.ContainerRun: - c = roaring.NewContainerRun(toInterval16(w)) - default: - panic(fmt.Sprintf("unknown container: %v", typ)) - } - c.SetMapped(true) - return c -} - -// StringifiedLMDBKeys returns a string with all the container -// keys available in lmdb. -func (w *LMDBWrapper) StringifiedLMDBKeys(optionalUseThisTx Tx, short bool) (r string) { - if optionalUseThisTx == nil { - tx, _ := w.NewTx(!writable, "", Txo{}) - defer tx.Rollback() - r = stringifiedLMDBKeysTx(tx.(*LMDBTx), short) - return - } - - btx, ok := optionalUseThisTx.(*LMDBTx) - if !ok { - return fmt.Sprintf("", optionalUseThisTx) - } - r = stringifiedLMDBKeysTx(btx, short) - return -} - -// countBitsSet returns the number of bits set (or "hot") in -// the roaring container value found by the txkey.Key() -// formatted bkey. -func (tx *LMDBTx) countBitsSet(bkey []byte) (n int) { - - //tx.debugOnlyGidcheck() - - v, err := tx.tx.Get(tx.dbi, bkey) - if lmdb.IsNotFound(err) { - // some queries bkey may not be present! don't panic. - return 0 - } - panicOn(err) - - n = len(v) - if n > 0 { - rc := tx.toContainer(v[n-1], v[0:(n-1)]) - n = int(rc.N()) - } - return -} - -func (tx *LMDBTx) Dump(short bool, shard uint64) { - fmt.Printf("%v\n", stringifiedLMDBKeysTx(tx, short)) -} - -// stringifiedLMDBKeysTx reports all the lmdb keys and a -// corresponding blake3 hash viewable by txn within the entire -// lmdb database. -// It also reports how many bits are hot in the roaring container -// (how many bits are set, or 1 rather than 0). -// -// By convention, we must return the empty string if there -// are no keys present. The tests use this to confirm -// an empty database. -func stringifiedLMDBKeysTx(tx *LMDBTx, short bool) (r string) { - - r = "allkeys:[\n" - it := NewLMDBIterator(tx, nil) - defer it.Close() - any := false - for it.Next() { - any = true - - bkey := it.lastKey - key := txkey.ToString(bkey) - ckey := txkey.KeyExtractContainerKey(bkey) - h := "" - srbm := "" - v := it.lastVal - n := len(v) - if n == 0 { - panic("should not have empty v here") - } - h = hash.Blake3sum16(v[0:(n - 1)]) - ct := tx.toContainer(v[n-1], v[0:(n-1)]) - cts := roaring.NewSliceContainers() - cts.Put(ckey, ct) - rbm := &roaring.Bitmap{Containers: cts} - srbm = BitmapAsString(rbm) - - r += fmt.Sprintf("%v -> %v (%v hot)\n", key, h, tx.countBitsSet(bkey)) - if !short { - r += " ......." + srbm + "\n" - } - } - r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) - - if !any { - return "" - } - return "lmdb-" + r -} - -func (w *LMDBWrapper) DeleteDBPath(dbs *DBShard) (err error) { - path := dbs.pathForType(lmdbTxn) - err = os.RemoveAll(path) - if err != nil { - return errors.Wrap(err, "DeleteDBPath") - } - // if we go back to flat instead of inside its own directory, - // there will be a second -lock file needing deletion too. - lockfile := path + "-lock" - if FileExists(lockfile) { - err = os.RemoveAll(lockfile) - } - return -} - -func (w *LMDBWrapper) DeleteField(index, field, fieldPath string) (err error) { - - // TODO(jea) cleanup: I think this fieldPath delete just goes away now. - // remove this commented stuff once we are sure. - // - // under blue-green roaring_lmdb, the directory will not be found, b/c roaring will have - // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: - // "If the path does not exist, RemoveAll returns nil (no error)" - - err = os.RemoveAll(fieldPath) - if err != nil { - return errors.Wrap(err, "removing directory") - } - prefix := txkey.FieldPrefix(index, field) - return w.DeletePrefix(prefix) -} - -func (w *LMDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { - prefix := txkey.Prefix(index, field, view, shard) - return w.DeletePrefix(prefix) -} - -func (w *LMDBWrapper) DeletePrefix(prefix []byte) error { - - tx, _ := w.NewTx(writable, w.name, Txo{}) - - // NewTx will grab these, so don't lock until after it. - w.muDb.Lock() - - bi := NewLMDBIterator(tx.(*LMDBTx), prefix) - - for bi.Next() { - err := bi.cur.Del(0) - if err != nil { - w.muDb.Unlock() - panic(err) - } - } - bi.Close() - - // Commit will grab the w.muDb lock, so we must release it first. - w.muDb.Unlock() - - err := tx.Commit() - panicOn(err) - - return nil -} - -func (tx *LMDBTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") - } - var buf bytes.Buffer - sz, err = rbm.WriteTo(&buf) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") - } - return ioutil.NopCloser(&buf), sz, err -} - -func (tx *LMDBTx) Options() Txo { - return tx.o -} - -// Sn retreives the serial number of the Tx. -func (tx *LMDBTx) Sn() int64 { - return tx.sn -} diff --git a/lmdb_other.go b/lmdb_other.go deleted file mode 100644 index 0768ecfc5..000000000 --- a/lmdb_other.go +++ /dev/null @@ -1,456 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !amd64 - -package pilosa - -// this is a stubbed out file to let 386 build. lmdb won't work well -// on 32-bit; not enough memory map address space. - -import ( - "fmt" - "io" - "strings" - "sync" - "time" - - rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg" - "github.com/pilosa/pilosa/v2/roaring" -) - -var _ = time.Now - -const isDebugRun = false -const TxInitialMmapSize = 1 << 30 // 1GB - -func ToContainer(typ byte, w []byte) (c *roaring.Container) { - panic("ToContainer not implemented yet on non-amd64") -} - -// lmdbRegistrar facilitates shutdown -// of all the lmdb databases started under -// tests. Its needed because most tests don't cleanup -// the *Index(es) they create. But we still -// want to shutdown lmdbDB goroutines -// after tests run. -// -// It also allows opening the same path twice to -// result in sharing the same open database handle, and -// thus the same transactional guarantees. -// -type lmdbRegistrar struct { - mu sync.Mutex - mp map[*LMDBWrapper]bool - - path2db map[string]*LMDBWrapper -} - -var globalLMDBReg *lmdbRegistrar = newLMDBTestRegistrar() - -func newLMDBTestRegistrar() *lmdbRegistrar { - - return &lmdbRegistrar{ - mp: make(map[*LMDBWrapper]bool), - path2db: make(map[string]*LMDBWrapper), - } -} - -func (r *lmdbRegistrar) OpenDBWrapper(path string, doAllocZero bool, rbfcfg *rbfcfg.Config) (DBWrapper, error) { - panic("lmdb only available on 64-bit arch") -} - -func (r *lmdbRegistrar) Size() int { - panic("lmdb only available on 64-bit arch") -} - -// register each lmdb created under tests, so we -// can clean them up. This is called by openLMDBWrapper() while -// holding the r.mu.Lock, since it needs to atomically -// check the registry and make a new instance only -// if one does not exist for its path, and otherwise -// return the existing instance. -func (r *lmdbRegistrar) unprotectedRegister(w *LMDBWrapper) { - panic("lmdb only available on 64-bit arch") -} - -// unregister removes w from r -func (r *lmdbRegistrar) unregister(w *LMDBWrapper) { - panic("lmdb only available on 64-bit arch") -} - -func DumpAllLMDB() { - panic("lmdb only available on 64-bit arch") -} - -// lmdbPath is a helper for determining the full directory -// in which the lmdb database will be stored. -func lmdbPath(path string) string { - if !strings.HasSuffix(path, "-lmdb") { - return path + "-lmdb" - } - return path -} - -// openLMDBDB opens the database in the bpath directoy -// without deleting any prior content. Any LMDBDB -// database directory will have the "-lmdb" suffix. -// -// openLMDBDB will check the registry and make a new instance only -// if one does not exist for its bpath. Otherwise it returns -// the existing instance. This insures only one lmdbDB -// per bpath in this pilosa node. -func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) { - panic("lmdb only available on 64-bit arch") -} - -var ErrShutdown = fmt.Errorf("shutting down") - -// DeleteIndex deletes all the containers associated with -// the named index from the lmdb database. -func (w *LMDBWrapper) DeleteIndex(indexName string) error { - panic("lmdb only available on 64-bit arch") -} - -// statically confirm that LMDBTx satisfies the Tx interface. -var _ Tx = (*LMDBTx)(nil) - -// LMDBWrapper provides the NewLMDBTx() method. -// Execute lmdbJob's via LMDBWrapper.submit(); these must -// be done by the lmdb goroutine worker pool. -type LMDBWrapper struct{} - -func (w *LMDBWrapper) IsClosed() bool { - panic("lmdb only available on 64-bit arch") -} - -// NewLMDBTx produces LMDB based ACID transactions. If -// the transaction will modify data, then the write flag must be true. -// Read-only queries should set write to false, to allow more concurrency. -// Methods on a LMDBTx are thread-safe, and can be called from -// different goroutines. -// -// initialIndexName is optional. It is set by the TxFactory from the Txo -// options provided at the Tx creation point. It allows us to recognize -// and isolate cross-index queries more quickly. It can always be empty "" -// but when set is highly useful for debugging. It has no impact -// on transaction behavior. -// -func (w *LMDBWrapper) NewLMDBTx(write bool, initialIndexName string, frag *fragment) (tx *LMDBTx) { - panic("lmdb only available on 64-bit arch") -} - -// Close shuts down the LMDB database. -func (w *LMDBWrapper) Close() (err error) { - panic("lmdb only available on 64-bit arch") -} - -// LMDBTx wraps a lmdb.Txn and provides the Tx interface -// method implementations. -// The methods on LMDBTx are thread-safe, and can be called -// from different goroutines. -type LMDBTx struct{} - -func (tx *LMDBTx) Type() string { - panic("lmdb only available on 64-bit arch") -} - -func (tx *LMDBTx) UseRowCache() bool { - panic("lmdb only available on 64-bit arch") -} - -func (tx *LMDBTx) Group() *TxGroup { - panic("lmdb only available on 64-bit arch") -} - -// Pointer gives us a memory address for the underlying transaction for debugging. -// It is public because we use it in roaring to report invalid container memory access -// outside of a transaction. -func (tx *LMDBTx) Pointer() string { - panic("lmdb only available on 64-bit arch") -} - -// Sn retreives the serial number of the Tx. -func (tx *LMDBTx) Sn() int64 { - panic("lmdb only available on 64-bit arch") -} - -// Rollback rolls back the transaction. -func (tx *LMDBTx) Rollback() { - panic("lmdb only available on 64-bit arch") -} - -// Commit commits the transaction to permanent storage. -// Commits can handle up to 100k updates to fragments -// at once, but not more. This is a LMDBDB imposed limit. -func (tx *LMDBTx) Commit() error { - panic("lmdb only available on 64-bit arch") -} - -// Readonly returns true iff the LMDBTx is read-only. -func (tx *LMDBTx) Readonly() bool { - panic("lmdb only available on 64-bit arch") -} - -// RoaringBitmap returns the roaring.Bitmap for all bits in the fragment. -func (tx *LMDBTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - panic("lmdb only available on 64-bit arch") -} - -// Container returns the requested roaring.Container, selected by fragment and ckey -func (tx *LMDBTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) { - panic("lmdb only available on 64-bit arch") -} - -// PutContainer stores rc under the specified fragment and container ckey. -func (tx *LMDBTx) PutContainer(index, field, view string, shard uint64, ckey uint64, rc *roaring.Container) error { - panic("lmdb only available on 64-bit arch") -} - -// RemoveContainer deletes the container specified by the shard and container key ckey -func (tx *LMDBTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error { - panic("lmdb only available on 64-bit arch") -} - -// Add sets all the a bits hot in the specified fragment. -func (tx *LMDBTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) { - panic("lmdb only available on 64-bit arch") - -} - -// Remove clears all the specified a bits in the chosen fragment. -func (tx *LMDBTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - panic("lmdb only available on 64-bit arch") -} - -// Contains returns exists true iff the bit chosen by key is -// hot (set to 1) in specified fragment. -func (tx *LMDBTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { - panic("lmdb only available on 64-bit arch") -} - -func (tx *LMDBTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { - panic("lmdb only available on 64-bit arch") -} - -// key is the container key for the first roaring Container -// roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will -// return the first container at or after key. found will be true if a -// container is found at key. -// -// LMDBTx notes: We auto-stop at the end of this shard, not going beyond. -func (tx *LMDBTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { - panic("lmdb only available on 64-bit arch") - -} - -// LMDBIterator is the iterator returned from a LMDBTx.ContainerIterator() call. -// It implements the roaring.ContainerIterator interface. -type LMDBIterator struct{} - -// NewLMDBIterator creates an iterator on tx that will -// only return lmdbKeys that start with prefix. -func NewLMDBIterator(tx *LMDBTx, prefix []byte) (bi *LMDBIterator) { - panic("lmdb only available on 64-bit arch") -} - -// Close tells the database and transaction that the user is done -// with the iterator. -func (bi *LMDBIterator) Close() { - panic("lmdb only available on 64-bit arch") -} - -// Valid returns false if there are no more values in the iterator's range. -func (bi *LMDBIterator) Valid() bool { - panic("lmdb only available on 64-bit arch") -} - -// Seek allows the iterator to start at needle instead of the global begining. -func (bi *LMDBIterator) Seek(needle []byte) (ok bool) { - panic("lmdb only available on 64-bit arch") -} - -func (bi *LMDBIterator) ValidForPrefix(prefix []byte) bool { - panic("lmdb only available on 64-bit arch") -} - -func (bi *LMDBIterator) String() (r string) { - panic("lmdb only available on 64-bit arch") -} - -var oneByteSliceOfZero = []byte{0} - -// Next advances the iterator. -func (bi *LMDBIterator) Next() (ok bool) { - panic("lmdb only available on 64-bit arch") -} - -// Value retrieves what is pointed at currently by the iterator. -func (bi *LMDBIterator) Value() (containerKey uint64, c *roaring.Container) { - panic("lmdb only available on 64-bit arch") -} - -// lmdbFinder implements roaring.IteratorFinder. -// It is used by LMDBTx.ForEach() -type lmdbFinder struct { - tx *LMDBTx - index string - field string - view string - shard uint64 - needClose []Closer -} - -// FindIterator lets lmdbFinder implement the roaring.FindIterator interface. -func (bf *lmdbFinder) FindIterator(seek uint64) (roaring.ContainerIterator, bool) { - panic("lmdb only available on 64-bit arch") -} - -// Close closes all bf.needClose listed Closers. -func (bf *lmdbFinder) Close() { - panic("lmdb only available on 64-bit arch") -} - -// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE -// the transaction Commits or Rollsback. -func (tx *LMDBTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { - panic("lmdb only available on 64-bit arch") -} - -// ForEach applies fn to each bitmap in the fragment. -func (tx *LMDBTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { - panic("lmdb only available on 64-bit arch") -} - -// ForEachRange applies fn on the selected range of bits on the chosen fragment. -func (tx *LMDBTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { - panic("lmdb only available on 64-bit arch") -} - -// Count operates on the full bitmap level, so it sums over all the containers -// in the bitmap. -func (tx *LMDBTx) Count(index, field, view string, shard uint64) (uint64, error) { - panic("lmdb only available on 64-bit arch") -} - -// Max is the maximum bit-value in your bitmap. -// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does. -func (tx *LMDBTx) Max(index, field, view string, shard uint64) (uint64, error) { - panic("lmdb only available on 64-bit arch") -} - -// Min returns the smallest bit set in the fragment. If no bit is hot, -// the second return argument is false. -func (tx *LMDBTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { - panic("lmdb only available on 64-bit arch") -} - -// CountRange returns the count of hot bits in the start, end range on the fragment. -// roaring.countRange counts the number of bits set between [start, end). -func (tx *LMDBTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { - panic("lmdb only available on 64-bit arch") -} - -// OffsetRange creates a new roaring.Bitmap to return in other. For all the -// hot bits in [start, endx) of the chosen fragment, it stores -// them into other but with offset added to their bit position. -// The primary client is doing this, using ShardWidth, already; see -// fragment.rowFromStorage() in fragment.go. For example: -// -// data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard, -// f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) -// ^ offset ^ start ^ endx -// -// The start and endx arguments are container keys that have been shifted left by 16 bits; -// their highbits() will be taken to determine the actual container keys. This -// is done to conform to the roaring.OffsetRange() argument convention. -// -func (tx *LMDBTx) OffsetRange(index, field, view string, shard, offset, start, endx uint64) (other *roaring.Bitmap, err error) { - panic("lmdb only available on 64-bit arch") -} - -// IncrementOpN increments the tx opcount by changedN -func (tx *LMDBTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - panic("lmdb only available on 64-bit arch") -} - -// ImportRoaringBits handles deletes by setting clear=true. -// rowSet[rowID] returns the number of bit changed on that rowID. -func (tx *LMDBTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { - panic("lmdb only available on 64-bit arch") -} - -func (tx *LMDBTx) toContainer(typ byte, v []byte) (r *roaring.Container) { - panic("lmdb only available on 64-bit arch") -} - -// StringifiedLMDBKeys returns a string with all the container -// keys available in lmdb. -func (w *LMDBWrapper) StringifiedLMDBKeys(optionalUseThisTx Tx) (r string) { - panic("lmdb only available on 64-bit arch") -} - -// countBitsSet returns the number of bits set (or "hot") in -// the roaring container value found by the txkey.Key() -// formatted bkey. -func (tx *LMDBTx) countBitsSet(bkey []byte) (n int) { - panic("lmdb only available on 64-bit arch") -} - -func (tx *LMDBTx) IsDone() (done bool) { - panic("lmdb only available on 64-bit arch") -} - -func (tx *LMDBTx) Dump(short bool, shard uint64) { - panic("lmdb only available on 64-bit arch") -} - -// stringifiedLMDBKeysTx reports all the lmdb keys and a -// corresponding blake3 hash viewable by txn within the entire -// lmdb database. -// It also reports how many bits are hot in the roaring container -// (how many bits are set, or 1 rather than 0). -// -// By convention, we must return the empty string if there -// are no keys present. The tests use this to confirm -// an empty database. -func stringifiedLMDBKeysTx(tx *LMDBTx) (r string) { - panic("lmdb only available on 64-bit arch") -} - -func (w *LMDBWrapper) DeleteField(index, field, fieldPath string) error { - panic("lmdb only available on 64-bit arch") -} - -func (w *LMDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { - panic("lmdb only available on 64-bit arch") -} - -func (w *LMDBWrapper) DeletePrefix(prefix []byte) error { - panic("lmdb only available on 64-bit arch") -} - -func (tx *LMDBTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - panic("lmdb only available on 64-bit arch") -} - -// UnionInPlace unions all the others Bitmaps into a new Bitmap, and then writes it to the -// specified fragment. -func (tx *LMDBTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - panic("lmdb only available on 64-bit arch") -} - -func (tx *LMDBTx) Options() Txo { - panic("lmdb only available on 64-bit arch") -} diff --git a/lmdb_test.go b/lmdb_test.go deleted file mode 100644 index 529c594a6..000000000 --- a/lmdb_test.go +++ /dev/null @@ -1,1294 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build amd64 - -package pilosa - -import ( - "fmt" - "math" - "os" - "testing" - - "github.com/pilosa/pilosa/v2/roaring" -) - -// helpers, each runs their own new txn, and commits if a change/delete -// was made. The txn is rolled back if it is just viewing the data. - -func LMDBMustHaveBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard uint64, bitvalue uint64) { - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue '%v' was NOT SET!!!", bitvalue)) - } - - tx.Rollback() -} - -func LMDBMustNotHaveBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard uint64, bitvalue uint64) { - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) - if exists { - panic(fmt.Sprintf("ARG bitvalue '%v' WAS SET but should not have been.!!!", bitvalue)) - } - tx.Rollback() -} - -func LMDBMustSetBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard uint64, putme uint64) { - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - // add a bit - changed, err := tx.Add(index, field, view, shard, doBatched, putme) - if changed != 1 { - panic("should have 1 bit changed") - } - panicOn(err) - - exists, err := tx.Contains(index, field, view, shard, putme) - panicOn(err) - if !exists { - panic("ARG putme was NOT SET!!!") - } - panicOn(tx.Commit()) -} - -func LMDBMustDeleteBitvalueContainer(dbwrap *LMDBWrapper, index, field, view string, shard uint64, putme uint64) { - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - hi := highbits(putme) - panicOn(tx.RemoveContainer(index, field, view, shard, hi)) - panicOn(tx.Commit()) -} - -func LMDBMustDeleteBitvalue(dbwrap *LMDBWrapper, index, field, view string, shard uint64, putme uint64) { - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - _, err := tx.Remove(index, field, view, shard, putme) - panicOn(err) - panicOn(tx.Commit()) -} - -func mustOpenEmptyLMDBWrapper(path string) (w *LMDBWrapper, cleaner func()) { - var err error - fn := lmdbPath(path) - panicOn(os.RemoveAll(fn)) - ww, err := globalLMDBReg.OpenDBWrapper(fn, DetectMemAccessPastTx, nil) - panicOn(err) - w = ww.(*LMDBWrapper) - - // verify it is empty - allkeys := w.StringifiedLMDBKeys(nil, false) - if allkeys != "" { - panic(fmt.Sprintf("freshly created database was not empty! had keys:'%v'", allkeys)) - } - - return w, func() { - w.Close() - panicOn(os.RemoveAll(fn)) - } -} - -// end of helper utilities -////////////////////////// - -////////////////////////// -// begin Tx method tests - -func TestLMDB_DeleteFragment(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLmdb_DeleteFragment") - defer clean() - defer dbwrap.Close() - index, field, view, shard0 := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - shard1 := uint64(1) - - bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} - shards := []uint64{shard0, shard1} - for _, s := range shards { - for _, v := range bits { - changed, err := tx.Add(index, field, view, s, doBatched, v) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - } - } - - for _, s := range shards { - for _, v := range bits { - exists, err := tx.Contains(index, field, view, s, v) - panicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - } - } - err := tx.Commit() - panicOn(err) - - // end of setup - - survivor := shard0 - victim := shard1 - err = dbwrap.DeleteFragment(index, field, view, victim, nil) - panicOn(err) - - tx, _ = dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - for _, s := range shards { - for _, v := range bits { - exists, err := tx.Contains(index, field, view, s, v) - panicOn(err) - if s == survivor { - if !exists { - panic(fmt.Sprintf("ARG survivor died : bit %v", v)) - } - } else if s == victim { // victim, should have been deleted - if exists { - panic(fmt.Sprintf("ARG victim lived : bit %v", v)) - } - } - } - } -} - -func TestLMDB_Max_on_many_containers(t *testing.T) { - path := "TestLMDB_Max_on_many_containers" - dbwrap, clean := mustOpenEmptyLMDBWrapper(path) - - defer clean() - defer dbwrap.Close() - index, field, view := "i", "f", "v" - - // 099 - // 101 - // 199 - // 300 - // 399 - // - // find max in [300,400) and get 399 - // find max in [000,100) and get 099 - // find max in [100,200) and get 199 - // find max in [400,500) and get nothing back - // find max in [200,300) and get nothing back - - shards := []int{99, 101, 199, 300, 399} - - for _, sh := range shards { - shard := uint64(sh) - for _, pm := range shards { - putme := uint64(pm) - if putme > shard { - continue - } - LMDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - } - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - for _, shard := range shards { - max, err := tx.Max(index, field, view, uint64(shard)) - panicOn(err) - if max != uint64(shard) { - panic(fmt.Sprintf("expected max (%v) to be == shard = %v", max, shard)) - } - } - - // check for not found - max, err := tx.Max(index, field, view, uint64(200)) - panicOn(err) - if max != 0 { - panic("expected not found to give 0 max back with nil err") - } - max, err = tx.Max(index, field, view, uint64(400)) - panicOn(err) - if max != 0 { - panic("expected not found to give 0 max back with nil err") - } - -} - -// and the rest - -func TestLMDB_SetBitmap(t *testing.T) { - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_SetBitmap") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - bitvalue := uint64(0) - changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - - err = tx.Commit() - panicOn(err) - - // - // commited, so should be visible outside the txn - // - - tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) - exists, err = tx2.Contains(index, field, view, shard, bitvalue) - panicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!! on tx2") - } - - n, err := tx2.Count(index, field, view, shard) - panicOn(err) - if n != 1 { - panic(fmt.Sprintf("should have Count 1; instead n = %v", n)) - } - tx2.Rollback() -} - -func TestLMDB_OffsetRange(t *testing.T) { - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_OffsetRange") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - bitvalue := uint64(1 << 20) - changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - - bitvalue2 := uint64(1<<20 + 1) - changed, err = tx.Add(index, field, view, shard, doBatched, bitvalue2) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - exists, err = tx.Contains(index, field, view, shard, bitvalue2) - panicOn(err) - if !exists { - panic("ARG bitvalue2 was NOT SET!!!") - } - - err = tx.Commit() - panicOn(err) - - offset := uint64(0 << 20) - start := uint64(0 << 16) - endx := bitvalue + 1<<16 - - tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) - rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx) - panicOn(err) - tx2.Rollback() - - // should see our 1M value - s2 := BitmapAsString(rbm2) - expect2 := "c(1048576, 1048577)" - if s2 != expect2 { - panic(fmt.Sprintf("s2='%v', but expected '%v'", s2, expect2)) - } - - // now offset by 2M - offset = uint64(2 << 20) - tx3, _ := dbwrap.NewTx(!writable, index, Txo{}) - rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx) - panicOn(err) - tx3.Rollback() - - //expect to see 3M == 3145728 - s3 := BitmapAsString(rbm3) - expect3 := "c(3145728, 3145729)" - - if s3 != expect3 { - panic(fmt.Sprintf("s3='%v', but expected '%v'", s3, expect3)) - } -} - -func TestLMDB_Count_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_Count_on_many_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{0, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - LMDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - n, err := tx.Count(index, field, view, shard) - panicOn(err) - if int(n) != len(putmeValues) { - panic(fmt.Sprintf("expected Count of %v but got n=%v", len(putmeValues), n)) - } -} - -func TestLMDB_Count_dense_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_Count_dense_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - expected := 0 - for i := uint64(0); i < (1<<16)+2; i += 2 { - changed, err := tx.Add(index, field, view, shard, doBatched, i) - panicOn(err) - if changed <= 0 { - panic("wat? should have changed") - } - expected++ - } - defer tx.Rollback() - - n, err := tx.Count(index, field, view, shard) - panicOn(err) - if int(n) != expected { - panic(fmt.Sprintf("expected Count of %v but got n=%v", expected, n)) - } -} - -func TestLMDB_ContainerIterator_on_empty(t *testing.T) { - // iterate on empty container, should not find anything. - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_ContainerIterator") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - bitvalue := uint64(0) - citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue) - panicOn(err) - defer citer.Close() - if found { - panic("should not have found anything") - } - panicOn(err) -} - -func TestLMDB_ContainerIterator_on_one_bit(t *testing.T) { - // set one bit, iterate. - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_ContainerIterator_on_one_bit") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - bitvalue := uint64(42) - - // add a bit - changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - - // same Tx, continues in use. - - citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(bitvalue)) - if !found { - panic("ContainerIterator did not find the 42 bit") - } - panicOn(err) - defer citer.Close() - - loopCount := 0 - for citer.Next() { - key, container := citer.Value() - if key != 0 { - panic("42 should have had key 0") - } - if container == nil { - panic("container was nil") - } - if container.N() != 1 { - panic("put a bit in, but size of container was not 1") - } - if !container.Contains(lowbits(bitvalue)) { - panic("container did not have our bitvalue!") - } - loopCount++ - if loopCount > 0 { // happier linter - break - } - } - if loopCount != 1 { - panic("ContainerIterator did not return a citer that scanned our set bit") - } -} - -func TestLMDB_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) { - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_ContainerIterator_on_one_bit_fail_to_find") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - putme := uint64(1<<16) + 3 // in the key:1 container - searchme := putme + 1 - - // add a bit - changed, err := tx.Add(index, field, view, shard, doBatched, putme) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - - exists, err := tx.Contains(index, field, view, shard, putme) - panicOn(err) - if !exists { - panic("ARG putme was NOT SET!!!") - } - - // same Tx, continues in use. - - citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(searchme)) - if !found { - panic("ContainerIterator did not find the searchme") - } - defer citer.Close() - loopCount := 0 - for citer.Next() { - key, container := citer.Value() - if key != 1 { - panic("Containeriterator searching for highbits(searchme) should not have had a bit") - } - if container == nil { - panic("container was nil") - } - if container.N() != 1 { - panic("put a bit in, but size of container was not 1") - } - if container.Contains(lowbits(searchme)) { - panic("container should have putme but not our searchme!") - } - loopCount++ - // only want first pass. keep linter happy by avoiding raw break - if loopCount > 0 { - break - } - } - panicOn(err) -} - -func TestLMDB_ContainerIterator_empty_iteration_loop(t *testing.T) { - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_ContainerIterator_empty_iteration_loop") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - putme := uint64(1<<16) + 3 // in the key:1 container - searchme := uint64(1 << 17) // in the next container, key:2 - - // add a bit - changed, err := tx.Add(index, field, view, shard, doBatched, putme) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - - exists, err := tx.Contains(index, field, view, shard, putme) - panicOn(err) - if !exists { - panic("ARG putme was NOT SET!!!") - } - - // same Tx, continues in use. - - citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(searchme)) - panicOn(err) - if found { - panic("ContainerIterator found the searchme, when it should not have") - } - defer citer.Close() - if citer.Next() { - panic("expected no looping, 0 iterations, b/c started searchme past our data in putme") - } - - // expect to see a blow up from the citer.Value() call, verify that we do. - func() { - defer func() { - r := recover() - if r == nil { - panic("expected a panic from citer.Value() in this case") - } - }() - citer.Value() // should panic - }() - -} - -func TestLMDB_ForEach_on_one_bit(t *testing.T) { - - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_ForEach_on_one_bit") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - bitvalue := uint64(42) - - // add a bit - changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - panicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - - // same Tx, continues in use. - count := 0 - err = tx.ForEach(index, field, view, shard, func(v uint64) error { - if v != bitvalue { - panic(fmt.Sprintf("bitvalue corrupt got %v want %v", v, bitvalue)) - } - count += 1 - return nil - }) - panicOn(err) - if count != 1 { - panic(fmt.Sprintf("Expected single iteration got %v ", count)) - } -} - -func TestLMDB_RemoveContainer_one_bit_test(t *testing.T) { - - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_RemoveContainer_one_bit_test") - defer clean() - defer dbwrap.Close() - - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{0, 13, 77, 1511} - - for _, putme := range putmeValues { - - // a) delete of whole container in a seperate txn. Commit should establish the deletion. - - LMDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - LMDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // b) deletion + rollback on the txn should restore the deleted bit - - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // delete, but rollback instead of commit - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - hi := highbits(putme) - panicOn(tx.RemoveContainer(index, field, view, shard, hi)) - tx.Rollback() - - // verify that the rollback undid the deletion. - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // c) within one Tx, after delete it should be gone as viewed within the txn. - tx, _ = dbwrap.NewTx(writable, index, Txo{}) - hi = highbits(putme) - - exists, err := tx.Contains(index, field, view, shard, putme) - panicOn(err) - if !exists { - panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme)) - } - - panicOn(tx.RemoveContainer(index, field, view, shard, hi)) - - exists, err = tx.Contains(index, field, view, shard, putme) - panicOn(err) - if exists { - panic(fmt.Sprintf("ARG putme '%v' was SET even after RemoveContiner in this txn.", putme)) - } - - tx.Rollback() - - // verify that the rollback undid the deletion. - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - // leave with clean slate - LMDBMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - } -} - -func TestLMDB_Remove_one_bit_test(t *testing.T) { - - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_Remove_one_bit_test") - defer clean() - defer dbwrap.Close() - - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{0, 13, 77, 1511} - - for _, putme := range putmeValues { - - // a) delete of whole container in a seperate txn. Commit should establish the deletion. - - LMDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustDeleteBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // b) deletion + rollback on the txn should restore the deleted bit - - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // delete, but rollback instead of commit - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - hi, lo := highbits(putme), lowbits(putme) - _, _ = hi, lo - _, err := tx.Remove(index, field, view, shard, hi) - panicOn(err) - tx.Rollback() - - // verify that the rollback undid the deletion. - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // c) within one Tx, after delete it should be gone as viewed within the txn. - tx, _ = dbwrap.NewTx(writable, index, Txo{}) - - exists, err := tx.Contains(index, field, view, shard, putme) - panicOn(err) - if !exists { - panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme)) - } - - mustRemove(tx.Remove(index, field, view, shard, putme)) - - exists, err = tx.Contains(index, field, view, shard, putme) - panicOn(err) - if exists { - panic(fmt.Sprintf("ARG putme '%v' was SET even after Remove in this txn.", putme)) - } - - tx.Rollback() - - // verify that the rollback undid the deletion. - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - // leave with clean slate - LMDBMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - } -} - -func TestLMDB_Min_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_Min_on_many_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - // verify no containers flag works - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - min, containersExist, err := tx.Min(index, field, view, shard) - _ = min - panicOn(err) - if containersExist { - panic("no containers should exist") - } - tx.Rollback() - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - LMDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ = dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - min, containersExist, err = tx.Min(index, field, view, shard) - panicOn(err) - if !containersExist { - panic("containers should exist") - } - expected := putmeValues[0] - if min != expected { - panic(fmt.Sprintf("expected Min() of %v but got min=%v", expected, min)) - } -} - -func TestLMDB_CountRange_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_CountRange_on_many_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - // verify no containers flag works - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64) - panicOn(err) - if n != 0 { - panic("no containers should exist") - } - tx.Rollback() - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - LMDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ = dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64) - panicOn(err) - if n == 0 { - panic("containers should exist") - } - expected := uint64(len(putmeValues)) - if n != expected { - panic(fmt.Sprintf("expected CountRange() of %v but got n=%v", expected, n)) - } -} - -func TestLMDB_CountRange_middle_container(t *testing.T) { - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_CountRange_middle_container") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - LMDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - // pick out just the middle container with the 1 bit set on it. - n, err := tx.CountRange(index, field, view, shard, 4, (2<<16)+1) - panicOn(err) - if n != 1 { - panic("middle 1 bit container should exist") - } -} - -func TestLMDB_CountRange_many_middle_container(t *testing.T) { - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_CountRange_many_middle_container") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - LMDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - // get them all - n, err := tx.CountRange(index, field, view, shard, 0, (4<<16)+1) - panicOn(err) - if n != 3 { - panic("count should have been all 3 bits") - } -} - -func TestLMDB_UnionInPlace(t *testing.T) { - - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_UnionInPlace") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{3, 2 << 16} - - others := roaring.NewBitmap() - others2 := roaring.NewBitmap() - others3 := roaring.NewBitmap() - // populate others with putmeValues +1 into others - - for _, putme := range putmeValues { - LMDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - LMDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) - n, err := tx2.Count(index, field, view, shard) - panicOn(err) - if n != 2 { - panic("should have 2 bits set") - } - tx2.Rollback() - - for _, putme := range putmeValues { - mustAddR(others.Add(putme)) // should not change count, b/c putme already in the rbm - mustAddR(others.Add(putme + 1)) - mustAddR(others2.Add(putme + 2)) - } - mustAddR(others3.Add(4 << 16)) // outside the 2<<16 container - - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - err = tx.UnionInPlace(index, field, view, shard, others, others2, others3) - panicOn(err) - - // end game, check we got the union. - rbm, err := tx.RoaringBitmap(index, field, view, shard) - panicOn(err) - n = rbm.Count() - if n != 7 { - panic("should have a total 3 + 3 +1 = 7 bits set on the containers") - } -} - -func TestLMDB_RoaringBitmap(t *testing.T) { - - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_RoaringBitmap") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - expected := uint64(3) - putme := expected - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - panicOn(err) - - slc := rbm.Slice() - if slc[0] != uint64(expected) { - panic(fmt.Sprintf("should have gotten %v back", expected)) - } -} - -func TestLMDB_ImportRoaringBits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_ImportRoaringBits") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - tx.(*LMDBTx).DeleteEmptyContainer = true // match Roaring - - //bitvalue := uint64(42) - - // get some roaring bits, get an itr RoaringIterator from them - rowSize := uint64(0) - //bits := []uint64{0} - bits := []uint64{0, 2, 5, 1<<16 + 1, 2 << 16} - data := getTestBitmapAsRawRoaring(bits...) - itr, err := roaring.NewRoaringIterator(data) - panicOn(err) - clear := false - logme := false - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != len(bits) { - panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) - } - panicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - panicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now test the union in place with the same set gives no change. - - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != 0 { - panic(fmt.Sprintf("should have not changed any bits on the second import, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - panicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - panicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now test the clear path - clear = true - - for _, v := range bits { - // clear 1 bit at a time - data := getTestBitmapAsRawRoaring(v) - itr, err := roaring.NewRoaringIterator(data) - panicOn(err) - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != 1 { - panic(fmt.Sprintf("should have changed 1 bit: '%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - panicOn(err) - } - n, err := tx.Count(index, field, view, shard) - panicOn(err) - if n != 0 { - panic(fmt.Sprintf("n = %v not zero so the clearbits didn't happen!", n)) - } - allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx), false) - - // should have no keys - if allkeys != "" { - panic("lmdb should have no keys now") - } -} - -func TestLMDB_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_ImportRoaringBits_set_nonoverlapping_bits") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - // get some roaring bits, get an itr RoaringIterator from them - rowSize := uint64(0) - //bits := []uint64{0} - bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} - data := getTestBitmapAsRawRoaring(bits...) - itr, err := roaring.NewRoaringIterator(data) - panicOn(err) - - bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16} - data2 := getTestBitmapAsRawRoaring(bits2...) - itr2, err := roaring.NewRoaringIterator(data2) - panicOn(err) - - clear := false - logme := false - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != len(bits) { - panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) - } - panicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - panicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now import the 2nd, overlapping set and set them. - - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil) - _ = rowSet - if changed != 4 { - panic(fmt.Sprintf("should have changed 2 bits: the 1 and the 3, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - panicOn(err) -} - -func TestLMDB_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_ImportRoaringBits_clear_nonoverlapping_bits") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - // get some roaring bits, get an itr RoaringIterator from them - rowSize := uint64(0) - //bits := []uint64{0} - bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} //, 5, 1<<16 + 1, 2 << 16} - data := getTestBitmapAsRawRoaring(bits...) - itr, err := roaring.NewRoaringIterator(data) - panicOn(err) - - bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16} - data2 := getTestBitmapAsRawRoaring(bits2...) - itr2, err := roaring.NewRoaringIterator(data2) - panicOn(err) - - clear := false - logme := false - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != len(bits) { - panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) - } - panicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - panicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now import the 2nd overlapping set and clear them. - clear = true - - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil) - _ = rowSet - if changed != 2 { - panic(fmt.Sprintf("should have changed 1 bit: the 2, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - panicOn(err) - - n, err := tx.Count(index, field, view, shard) - panicOn(err) - if n != 2 { // just the 0 and the 1<<16 bits should be left set. - panic(fmt.Sprintf("n = %v not 2 so the clearbits didn't happen!", n)) - } - -} - -func TestLMDB_DeleteIndex(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_DeleteIndex") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - bitvalue := uint64(777) - bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} - for _, v := range bits { - changed, err := tx.Add(index, field, view, shard, doBatched, v) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - } - - index2 := "i2" // should not be deleted, even though it shares a prefix with 'i' - changed, err := tx.Add(index2, field, view, shard, doBatched, bitvalue) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - panicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - } - exists, err := tx.Contains(index2, field, view, shard, bitvalue) - panicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!! on index2") - } - err = tx.Commit() - panicOn(err) - - // end of setup - err = dbwrap.DeleteIndex(index) - panicOn(err) - - tx, _ = dbwrap.NewTx(!writable, index2, Txo{}) - defer tx.Rollback() - exists, err = tx.Contains(index2, field, view, shard, bitvalue) - panicOn(err) - if !exists { - panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2)) - } - - for _, v := range bits { - exists, err = tx.Contains(index, field, view, shard, v) - panicOn(err) - if exists { - allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx), false) - panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) - } - } -} - -func TestLMDB_DeleteIndex_over100k(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_DeleteIndex_over100k") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - bitvalue := uint64(777) - limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction. - //limit := uint64(101) - for v := uint64(1); v < limit; v++ { - // shift by << 16 to get into a different shard - changed, err := tx.Add(index, field, view, shard, doBatched, v<<16) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - if v%100000 == 0 { - panicOn(tx.Commit()) - tx, _ = dbwrap.NewTx(writable, index, Txo{}) - } - } - - index2 := "i2" // should not be deleted, even though it shares a prefix with 'i' - changed, err := tx.Add(index2, field, view, shard, doBatched, bitvalue) - if changed <= 0 { - panic("should have changed") - } - panicOn(err) - err = tx.Commit() - panicOn(err) - - // end of setup - err = dbwrap.DeleteIndex(index) - panicOn(err) - - tx, _ = dbwrap.NewTx(!writable, index2, Txo{}) - defer tx.Rollback() - exists, err := tx.Contains(index2, field, view, shard, bitvalue) - panicOn(err) - if !exists { - panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2)) - } - - for v := uint64(0); v < limit; v++ { - exists, err = tx.Contains(index, field, view, shard, v<<16) - panicOn(err) - if exists { - allkeys := stringifiedLMDBKeysTx(tx.(*LMDBTx), false) - panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) - } - } -} - -func TestLMDB_SliceOfShards(t *testing.T) { - - dbwrap, clean := mustOpenEmptyLMDBWrapper("TestLMDB_SliceOfShards") - defer clean() - defer dbwrap.Close() - index, field, view := "i", "f", "v" - shards := []uint64{0, 1, 2, 3, 1000001, 2000001} - putme := uint64(179) - for _, shard := range shards { - LMDBMustSetBitvalue(dbwrap, index, field, view, shard, putme) - } - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - slc, err := tx.SliceOfShards(index, field, view, "") - panicOn(err) - for i := range shards { - if shards[i] != slc[i] { - panic(fmt.Sprintf("expected at i=%v that slc[i]=%v = shards[i]=%v", i, slc[i], shards[i])) - } - } -} - -func TestLMDB_HasData(t *testing.T) { - - db, clean := mustOpenEmptyLMDBWrapper("TestLMDB_SliceOfShards") - defer clean() - defer db.Close() - - // HasData should start out false. - hasAnything, err := db.HasData() - if err != nil { - t.Fatal(err) - } - if hasAnything { - t.Fatalf("HasData reported existing data on an empty database") - } - - // check that HasData sees a committed record. - - index, field, view, shard, putme := "i", "f", "v", uint64(123), uint64(42) - LMDBMustSetBitvalue(db, index, field, view, shard, putme) - - // HasData(false) should now report data - hasAnything, err = db.HasData() - if err != nil { - t.Fatal(err) - } - if !hasAnything { - t.Fatalf("HasData() reported no data on a database that has bits written to it") - } -} diff --git a/txfactory.go b/txfactory.go index 03fa34972..768c065c6 100644 --- a/txfactory.go +++ b/txfactory.go @@ -38,7 +38,6 @@ import ( // public strings that pilosa/server/config.go can reference const ( RoaringTxn string = "roaring" - LmdbTxn string = "lmdb" RBFTxn string = "rbf" BoltTxn string = "bolt" ) @@ -53,7 +52,7 @@ const DefaultTxsrc = RBFTxn // which the transaction has committed or rolled back. Since // memory segments will be recycled by the underlying databases, // this can lead to corruption. When DetectMemAccessPastTx is true, -// code in lmdb.go will copy the transactionally viewed memory before +// code in bolt.go will copy the transactionally viewed memory before // returning it for bitmap reading, and then zero it or overwrite it // with -2 when the Tx completes. // @@ -83,8 +82,7 @@ var sep = string(os.PathSeparator) // course that your "new" read Tx actually has an "old" view // of the database. // -// At the moment, given that LMDB demands that -// all write Tx are created and executed on the same C thread, most +// At the moment, most // writes to individual shards are commited eagerly and locally // when the `defer finisher(&err0)` is run. // This is done by returning a finisher that actually Commits, @@ -265,16 +263,8 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) { // don't deadlock against themselves under blue-green. o.Write = o.Write || qcx.write - // note: write Tx were re-using Tx across different goroutines, - // which lmdb will not be pleased with. For reads this - // should be okay, as the docs say - // "If you want to pass read-only transactions across threads, - // you can use the MDB_NOTLS option on the environment." - // -- http://www.lmdb.tech/doc/starting.html - // and we always use lmdb.NoTLS as the lmdb-go bindings ensure this. - // - // So we make ALL write transactions local, and never reuse them - // below. + // In general, we make ALL write transactions local, and never reuse them + // below. Previously this was to help lmdb. // // *However* there is one exception: when we have set RequiredForAtomicWriteTx // for the importing of an AtomicRequest, then we must use that @@ -400,7 +390,7 @@ func (qcx *Qcx) ListOpenTx() string { } // TxFactory abstracts the creation of Tx interface-level -// transactions so that RBF, BoltDB, LMDB, or Roaring-fragment-files, or several +// transactions so that RBF, BoltDB, or Roaring-fragment-files, or several // of these at once in parallel, is used as the storage and transction layer. type TxFactory struct { typeOfTx string @@ -435,13 +425,12 @@ const ( noneTxn txtype = 0 roaringTxn txtype = 1 // these don't really have any transactions rbfTxn txtype = 2 - lmdbTxn txtype = 3 boltTxn txtype = 4 ) // these need to be skipped by the holder.go field scanner that // calls IsTxDatabasePath -var allTypesWithSuffixes = []txtype{rbfTxn, lmdbTxn, boltTxn} +var allTypesWithSuffixes = []txtype{rbfTxn, boltTxn} // FileSuffix is used to determine backend directory names. // We append '@' to be sure we never collide with a field name @@ -454,8 +443,6 @@ func (ty txtype) FileSuffix() string { return "" case rbfTxn: return "-rbfdb@" - case lmdbTxn: - return "-lmdb@" case boltTxn: return "-boltdb@" } @@ -504,8 +491,6 @@ func MustTxsrcToTxtype(txsrc string) (types []txtype) { types = append(types, roaringTxn) case RBFTxn: // "rbf" types = append(types, rbfTxn) - case LmdbTxn: // "lmdb" - types = append(types, lmdbTxn) case BoltTxn: // "bolt" types = append(types, boltTxn) default: @@ -886,8 +871,6 @@ func (ty txtype) String() string { return "roaring" case rbfTxn: return "rbf" - case lmdbTxn: - return "lmdb" case boltTxn: return "bolt" } @@ -1253,9 +1236,6 @@ func anyGlobalDBWrappersStillOpen() bool { if globalRbfDBReg.Size() != 0 { return true } - if globalLMDBReg.Size() != 0 { - return true - } if globalBoltReg.Size() != 0 { return true } diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go index 580e2956d..2f4b62ee3 100644 --- a/txfactory_internal_test.go +++ b/txfactory_internal_test.go @@ -22,7 +22,6 @@ import ( "time" "github.com/glycerine/lmdb-go/lmdb" - //"github.com/pilosa/pilosa/v2/logger" ) func Test_TxFactory_Qcx_query_context(t *testing.T) { @@ -121,7 +120,7 @@ func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) { orig := os.Getenv("PILOSA_TXSRC") defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests! - checked := []string{"lmdb", "roaring", "rbf"} + checked := []string{"roaring", "rbf"} expectError := false for _, blue := range checked { @@ -270,7 +269,7 @@ func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) { orig := os.Getenv("PILOSA_TXSRC") defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests! - checked := []string{"lmdb", "roaring", "bolt", "rbf"} + checked := []string{"roaring", "bolt", "rbf"} for _, blue := range checked { for _, green := range checked { @@ -408,8 +407,8 @@ func Test_TxFactory_verifyStringConstantsMatch(t *testing.T) { // our const definitions at the top of txfactory.go, or // else blue-green transactions cannot determine when // the second transaction is being released in dbshard.go. - check := []txtype{roaringTxn, rbfTxn, lmdbTxn, boltTxn} - expect := []string{RoaringTxn, RBFTxn, LmdbTxn, BoltTxn} + check := []txtype{roaringTxn, rbfTxn, boltTxn} + expect := []string{RoaringTxn, RBFTxn, BoltTxn} for i, chk := range check { obs := chk.String() if obs != expect[i] { diff --git a/vprint_test.go b/vprint_test.go index 7e531e26d..a44a02ad9 100644 --- a/vprint_test.go +++ b/vprint_test.go @@ -51,6 +51,7 @@ func init() { // keeper linter happy _ = pp _ = vv + _ = DirExists } func PanicOn(err error) {