mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-09 14:41:02 +00:00
Merge pull request #1183 from jaten-molecula/rm_lmdb
remove lmdb as tx backend
This commit is contained in:
jaten-molecula
GitHub
commit
d38184a5ff
15 changed files with 76 additions and 3610 deletions
|
|
@ -208,7 +208,7 @@ workflows:
|
|||
- setup
|
||||
matrix:
|
||||
parameters:
|
||||
test_make_target: ["test-race", "test-txstore-rbf", "test-txstore-rbf_lmdb"]
|
||||
test_make_target: ["test-race", "test-txstore-rbf", "test-txstore-rbf_bolt"]
|
||||
- test:
|
||||
name: test-shardwidth-22
|
||||
shard_width: "22"
|
||||
|
|
|
|||
6
Makefile
6
Makefile
|
|
@ -1,4 +1,4 @@
|
|||
.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf_lmdb test-txstore-rbf
|
||||
.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf
|
||||
|
||||
CLONE_URL=github.com/pilosa/pilosa
|
||||
MOD_VERSION=v2
|
||||
|
|
@ -309,6 +309,6 @@ install-gometalinter:
|
|||
test-txstore-rbf:
|
||||
PILOSA_TXSRC=rbf $(MAKE) testv-race
|
||||
|
||||
test-txstore-rbf_lmdb:
|
||||
PILOSA_TXSRC=rbf_lmdb $(MAKE) testv-race
|
||||
test-txstore-rbf_bolt:
|
||||
PILOSA_TXSRC=rbf_bolt $(MAKE) testv-race
|
||||
|
||||
|
|
|
|||
17
bolt.go
17
bolt.go
|
|
@ -37,6 +37,8 @@ import (
|
|||
bolt "go.etcd.io/bbolt"
|
||||
)
|
||||
|
||||
const isDebugRun = false
|
||||
|
||||
// boltRegistrar facilitates shutdown
|
||||
// of all the bolt databases started under
|
||||
// tests. Its needed because most tests don't cleanup
|
||||
|
|
@ -1419,6 +1421,21 @@ func (tx *BoltTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
|
|||
return ToContainer(typ, w)
|
||||
}
|
||||
|
||||
func ToContainer(typ byte, w []byte) (c *roaring.Container) {
|
||||
switch typ {
|
||||
case roaring.ContainerArray:
|
||||
c = roaring.NewContainerArray(toArray16(w))
|
||||
case roaring.ContainerBitmap:
|
||||
c = roaring.NewContainerBitmap(-1, toArray64(w))
|
||||
case roaring.ContainerRun:
|
||||
c = roaring.NewContainerRun(toInterval16(w))
|
||||
default:
|
||||
panic(fmt.Sprintf("unknown container: %v", typ))
|
||||
}
|
||||
c.SetMapped(true)
|
||||
return c
|
||||
}
|
||||
|
||||
// StringifiedBoltKeys returns a string with all the container
|
||||
// keys available in bolt.
|
||||
func (w *BoltWrapper) StringifiedBoltKeys(optionalUseThisTx Tx, short bool) (r string) {
|
||||
|
|
|
|||
19
const_amd64.go
Normal file
19
const_amd64.go
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
// Copyright 2020 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build amd64
|
||||
|
||||
package pilosa
|
||||
|
||||
const TxInitialMmapSize = 4 << 30 // 4GB
|
||||
21
const_other.go
Normal file
21
const_other.go
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
// Copyright 2020 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build !amd64
|
||||
|
||||
package pilosa
|
||||
|
||||
// this is a stubbed out file to let 386/arm build.
|
||||
|
||||
const TxInitialMmapSize = 1 << 30 // 1GB
|
||||
|
|
@ -478,7 +478,6 @@ func (dbs *DBShard) DumpAll() {
|
|||
switch ty {
|
||||
case roaringTxn:
|
||||
case rbfTxn:
|
||||
case lmdbTxn:
|
||||
case boltTxn:
|
||||
default:
|
||||
panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
|
||||
|
|
@ -605,8 +604,6 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *
|
|||
registry = globalRoaringReg
|
||||
case rbfTxn:
|
||||
registry = globalRbfDBReg
|
||||
case lmdbTxn:
|
||||
registry = globalLMDBReg
|
||||
case boltTxn:
|
||||
registry = globalBoltReg
|
||||
default:
|
||||
|
|
@ -888,9 +885,9 @@ func listDirUnderDir(root string, includeRoot bool, requiredSuffix string, ignor
|
|||
// The blue is the destination -- this is always types[0].
|
||||
// The green source is always types[1]. The mnemonic is blue_geen.
|
||||
// The blue is first, so it is in types[0]. The green
|
||||
// is second, in types[1]. For example, with PILOSA_TXSRC=lmdb_roaring
|
||||
// we have lmdb as blue, and roaring as green. The contents of
|
||||
// lmdb must be empty or exactly match roaring. If lmdb
|
||||
// is second, in types[1]. For example, with PILOSA_TXSRC=bolt_roaring
|
||||
// we have bolt as blue, and roaring as green. The contents of
|
||||
// bolt must be empty or exactly match roaring. If bolt
|
||||
// starts empty, it will be populated from roaring by
|
||||
// populateBlueFromGreen().
|
||||
//
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
|
|||
orig := os.Getenv("PILOSA_TXSRC")
|
||||
defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
|
||||
|
||||
for _, src := range []string{"lmdb", "roaring", "bolt", "rbf"} {
|
||||
for _, src := range []string{"roaring", "bolt", "rbf"} {
|
||||
|
||||
os.Setenv("PILOSA_TXSRC", src)
|
||||
|
||||
|
|
@ -131,14 +131,6 @@ rick/_exists/views/standard/fragments/217
|
|||
rick/_exists/views/standard/fragments/93
|
||||
rick/_exists/views/standard/fragments/219
|
||||
rick/_exists/views/standard/fragments/223
|
||||
`,
|
||||
"lmdb": `
|
||||
rick.index.txstores@@@/store-lmdb@@/shard.0093-lmdb@
|
||||
rick.index.txstores@@@/store-lmdb@@/shard.0215-lmdb@
|
||||
rick.index.txstores@@@/store-lmdb@@/shard.0217-lmdb@
|
||||
rick.index.txstores@@@/store-lmdb@@/shard.0219-lmdb@
|
||||
rick.index.txstores@@@/store-lmdb@@/shard.0221-lmdb@
|
||||
rick.index.txstores@@@/store-lmdb@@/shard.0223-lmdb@
|
||||
`,
|
||||
"bolt": `
|
||||
rick.index.txstores@@@/store-boltdb@@/shard.0093-boltdb@/bolt.db
|
||||
|
|
@ -173,13 +165,6 @@ func makeSampleRoaringDir(root, txsrc string, minBytes int, h *Holder) {
|
|||
shard = shards[i]
|
||||
}
|
||||
switch txsrc {
|
||||
case "lmdb":
|
||||
makeLMDBtestDB(root+sep+fn, h, shard)
|
||||
// also have to make the DBShard in our in-memory tree,
|
||||
// or else the search won't find it because
|
||||
// DBPerShard won't know anything about it.
|
||||
helperCreateDBShard(h, index, shard)
|
||||
continue
|
||||
case "bolt":
|
||||
makeBolttestDB(root+sep+fn, h, shard)
|
||||
helperCreateDBShard(h, index, shard)
|
||||
|
|
@ -210,14 +195,6 @@ func helperCreateDBShard(h *Holder, index string, shard uint64) {
|
|||
_ = dbs
|
||||
}
|
||||
|
||||
func makeLMDBtestDB(path string, h *Holder, shard uint64) {
|
||||
i := uint64(1)
|
||||
w, _ := mustOpenEmptyLMDBWrapper(path)
|
||||
LMDBMustSetBitvalue(w, "index", "field", "view", shard, i)
|
||||
w.Close()
|
||||
|
||||
}
|
||||
|
||||
func makeBolttestDB(path string, h *Holder, shard uint64) {
|
||||
i := uint64(1)
|
||||
w, _ := mustOpenEmptyBoltWrapper(path)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ package pilosa_test
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
|
|
@ -28,59 +27,6 @@ import (
|
|||
"github.com/pilosa/pilosa/v2/test"
|
||||
)
|
||||
|
||||
func skipForNonLMDB(t *testing.T) {
|
||||
src := os.Getenv("PILOSA_TXSRC")
|
||||
if src != "lmdb" {
|
||||
t.Skip("skip if not lmdb")
|
||||
}
|
||||
}
|
||||
|
||||
var _ = skipForNonLMDB // happy linter
|
||||
|
||||
// Can't write it all to one shard like we do (did).
|
||||
func Test_DBPerShard_multiple_shards_used(t *testing.T) {
|
||||
skipForNonLMDB(t)
|
||||
c := test.MustRunCluster(t, 1)
|
||||
defer c.Close()
|
||||
hldr := c.GetHolder(0)
|
||||
index := "i"
|
||||
hldr.SetBit(index, "general", 10, 0)
|
||||
hldr.SetBit(index, "general", 10, ShardWidth+1)
|
||||
hldr.SetBit(index, "general", 10, ShardWidth+2)
|
||||
|
||||
hldr.SetBit(index, "general", 11, 2)
|
||||
hldr.SetBit(index, "general", 11, ShardWidth+2)
|
||||
|
||||
types := pilosa.MustTxsrcToTxtype("lmdb")
|
||||
idx := hldr.Index(index)
|
||||
shardsU := []uint64{0, 1, 2}
|
||||
pathShard := []string{}
|
||||
|
||||
// check that 3 different shard databases/files were made
|
||||
for i := 0; i < 2; i++ {
|
||||
|
||||
path, err := hldr.Txf().GetDBShardPath(index, shardsU[i], idx, types[0], !writable)
|
||||
panicOn(err)
|
||||
pathShard = append(pathShard, path)
|
||||
|
||||
if !DirExists(pathShard[i]) {
|
||||
panic(fmt.Sprintf("no shard made for pathShard[%v]='%v'", i, pathShard[i]))
|
||||
}
|
||||
sz, err := pilosa.DiskUse(pathShard[i], "")
|
||||
panicOn(err)
|
||||
|
||||
if sz < 100 {
|
||||
panic(fmt.Sprintf("shard %v was too small", i))
|
||||
}
|
||||
}
|
||||
|
||||
if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: `Union(Row(general=10), Row(general=11))`}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if columns := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, ShardWidth + 1, ShardWidth + 2}) {
|
||||
t.Fatalf("unexpected columns: %+v", columns)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
|
||||
|
||||
c := test.MustRunCluster(t, 1,
|
||||
|
|
|
|||
|
|
@ -5110,9 +5110,6 @@ func TestImportValueConcurrent(t *testing.T) {
|
|||
"blueGreenTx because the lack of transactional consistency " +
|
||||
"from Roaring-per-file will create false comparison " +
|
||||
"failures."))
|
||||
case lmdbTxn:
|
||||
t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " +
|
||||
"lmdb since only a single writer is allowed at once."))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
456
lmdb_other.go
456
lmdb_other.go
|
|
@ -1,456 +0,0 @@
|
|||
// Copyright 2020 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build !amd64
|
||||
|
||||
package pilosa
|
||||
|
||||
// this is a stubbed out file to let 386 build. lmdb won't work well
|
||||
// on 32-bit; not enough memory map address space.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg"
|
||||
"github.com/pilosa/pilosa/v2/roaring"
|
||||
)
|
||||
|
||||
var _ = time.Now
|
||||
|
||||
const isDebugRun = false
|
||||
const TxInitialMmapSize = 1 << 30 // 1GB
|
||||
|
||||
func ToContainer(typ byte, w []byte) (c *roaring.Container) {
|
||||
panic("ToContainer not implemented yet on non-amd64")
|
||||
}
|
||||
|
||||
// lmdbRegistrar facilitates shutdown
|
||||
// of all the lmdb databases started under
|
||||
// tests. Its needed because most tests don't cleanup
|
||||
// the *Index(es) they create. But we still
|
||||
// want to shutdown lmdbDB goroutines
|
||||
// after tests run.
|
||||
//
|
||||
// It also allows opening the same path twice to
|
||||
// result in sharing the same open database handle, and
|
||||
// thus the same transactional guarantees.
|
||||
//
|
||||
type lmdbRegistrar struct {
|
||||
mu sync.Mutex
|
||||
mp map[*LMDBWrapper]bool
|
||||
|
||||
path2db map[string]*LMDBWrapper
|
||||
}
|
||||
|
||||
var globalLMDBReg *lmdbRegistrar = newLMDBTestRegistrar()
|
||||
|
||||
func newLMDBTestRegistrar() *lmdbRegistrar {
|
||||
|
||||
return &lmdbRegistrar{
|
||||
mp: make(map[*LMDBWrapper]bool),
|
||||
path2db: make(map[string]*LMDBWrapper),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *lmdbRegistrar) OpenDBWrapper(path string, doAllocZero bool, rbfcfg *rbfcfg.Config) (DBWrapper, error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (r *lmdbRegistrar) Size() int {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// register each lmdb created under tests, so we
|
||||
// can clean them up. This is called by openLMDBWrapper() while
|
||||
// holding the r.mu.Lock, since it needs to atomically
|
||||
// check the registry and make a new instance only
|
||||
// if one does not exist for its path, and otherwise
|
||||
// return the existing instance.
|
||||
func (r *lmdbRegistrar) unprotectedRegister(w *LMDBWrapper) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// unregister removes w from r
|
||||
func (r *lmdbRegistrar) unregister(w *LMDBWrapper) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func DumpAllLMDB() {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// lmdbPath is a helper for determining the full directory
|
||||
// in which the lmdb database will be stored.
|
||||
func lmdbPath(path string) string {
|
||||
if !strings.HasSuffix(path, "-lmdb") {
|
||||
return path + "-lmdb"
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
// openLMDBDB opens the database in the bpath directoy
|
||||
// without deleting any prior content. Any LMDBDB
|
||||
// database directory will have the "-lmdb" suffix.
|
||||
//
|
||||
// openLMDBDB will check the registry and make a new instance only
|
||||
// if one does not exist for its bpath. Otherwise it returns
|
||||
// the existing instance. This insures only one lmdbDB
|
||||
// per bpath in this pilosa node.
|
||||
func (r *lmdbRegistrar) openLMDBWrapper(path0 string) (*LMDBWrapper, error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
var ErrShutdown = fmt.Errorf("shutting down")
|
||||
|
||||
// DeleteIndex deletes all the containers associated with
|
||||
// the named index from the lmdb database.
|
||||
func (w *LMDBWrapper) DeleteIndex(indexName string) error {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// statically confirm that LMDBTx satisfies the Tx interface.
|
||||
var _ Tx = (*LMDBTx)(nil)
|
||||
|
||||
// LMDBWrapper provides the NewLMDBTx() method.
|
||||
// Execute lmdbJob's via LMDBWrapper.submit(); these must
|
||||
// be done by the lmdb goroutine worker pool.
|
||||
type LMDBWrapper struct{}
|
||||
|
||||
func (w *LMDBWrapper) IsClosed() bool {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// NewLMDBTx produces LMDB based ACID transactions. If
|
||||
// the transaction will modify data, then the write flag must be true.
|
||||
// Read-only queries should set write to false, to allow more concurrency.
|
||||
// Methods on a LMDBTx are thread-safe, and can be called from
|
||||
// different goroutines.
|
||||
//
|
||||
// initialIndexName is optional. It is set by the TxFactory from the Txo
|
||||
// options provided at the Tx creation point. It allows us to recognize
|
||||
// and isolate cross-index queries more quickly. It can always be empty ""
|
||||
// but when set is highly useful for debugging. It has no impact
|
||||
// on transaction behavior.
|
||||
//
|
||||
func (w *LMDBWrapper) NewLMDBTx(write bool, initialIndexName string, frag *fragment) (tx *LMDBTx) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Close shuts down the LMDB database.
|
||||
func (w *LMDBWrapper) Close() (err error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// LMDBTx wraps a lmdb.Txn and provides the Tx interface
|
||||
// method implementations.
|
||||
// The methods on LMDBTx are thread-safe, and can be called
|
||||
// from different goroutines.
|
||||
type LMDBTx struct{}
|
||||
|
||||
func (tx *LMDBTx) Type() string {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (tx *LMDBTx) UseRowCache() bool {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (tx *LMDBTx) Group() *TxGroup {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Pointer gives us a memory address for the underlying transaction for debugging.
|
||||
// It is public because we use it in roaring to report invalid container memory access
|
||||
// outside of a transaction.
|
||||
func (tx *LMDBTx) Pointer() string {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Sn retreives the serial number of the Tx.
|
||||
func (tx *LMDBTx) Sn() int64 {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Rollback rolls back the transaction.
|
||||
func (tx *LMDBTx) Rollback() {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Commit commits the transaction to permanent storage.
|
||||
// Commits can handle up to 100k updates to fragments
|
||||
// at once, but not more. This is a LMDBDB imposed limit.
|
||||
func (tx *LMDBTx) Commit() error {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Readonly returns true iff the LMDBTx is read-only.
|
||||
func (tx *LMDBTx) Readonly() bool {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// RoaringBitmap returns the roaring.Bitmap for all bits in the fragment.
|
||||
func (tx *LMDBTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Container returns the requested roaring.Container, selected by fragment and ckey
|
||||
func (tx *LMDBTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// PutContainer stores rc under the specified fragment and container ckey.
|
||||
func (tx *LMDBTx) PutContainer(index, field, view string, shard uint64, ckey uint64, rc *roaring.Container) error {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// RemoveContainer deletes the container specified by the shard and container key ckey
|
||||
func (tx *LMDBTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Add sets all the a bits hot in the specified fragment.
|
||||
func (tx *LMDBTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
|
||||
}
|
||||
|
||||
// Remove clears all the specified a bits in the chosen fragment.
|
||||
func (tx *LMDBTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Contains returns exists true iff the bit chosen by key is
|
||||
// hot (set to 1) in specified fragment.
|
||||
func (tx *LMDBTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (tx *LMDBTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// key is the container key for the first roaring Container
|
||||
// roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will
|
||||
// return the first container at or after key. found will be true if a
|
||||
// container is found at key.
|
||||
//
|
||||
// LMDBTx notes: We auto-stop at the end of this shard, not going beyond.
|
||||
func (tx *LMDBTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
|
||||
}
|
||||
|
||||
// LMDBIterator is the iterator returned from a LMDBTx.ContainerIterator() call.
|
||||
// It implements the roaring.ContainerIterator interface.
|
||||
type LMDBIterator struct{}
|
||||
|
||||
// NewLMDBIterator creates an iterator on tx that will
|
||||
// only return lmdbKeys that start with prefix.
|
||||
func NewLMDBIterator(tx *LMDBTx, prefix []byte) (bi *LMDBIterator) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Close tells the database and transaction that the user is done
|
||||
// with the iterator.
|
||||
func (bi *LMDBIterator) Close() {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Valid returns false if there are no more values in the iterator's range.
|
||||
func (bi *LMDBIterator) Valid() bool {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Seek allows the iterator to start at needle instead of the global begining.
|
||||
func (bi *LMDBIterator) Seek(needle []byte) (ok bool) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (bi *LMDBIterator) ValidForPrefix(prefix []byte) bool {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (bi *LMDBIterator) String() (r string) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
var oneByteSliceOfZero = []byte{0}
|
||||
|
||||
// Next advances the iterator.
|
||||
func (bi *LMDBIterator) Next() (ok bool) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Value retrieves what is pointed at currently by the iterator.
|
||||
func (bi *LMDBIterator) Value() (containerKey uint64, c *roaring.Container) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// lmdbFinder implements roaring.IteratorFinder.
|
||||
// It is used by LMDBTx.ForEach()
|
||||
type lmdbFinder struct {
|
||||
tx *LMDBTx
|
||||
index string
|
||||
field string
|
||||
view string
|
||||
shard uint64
|
||||
needClose []Closer
|
||||
}
|
||||
|
||||
// FindIterator lets lmdbFinder implement the roaring.FindIterator interface.
|
||||
func (bf *lmdbFinder) FindIterator(seek uint64) (roaring.ContainerIterator, bool) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Close closes all bf.needClose listed Closers.
|
||||
func (bf *lmdbFinder) Close() {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE
|
||||
// the transaction Commits or Rollsback.
|
||||
func (tx *LMDBTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// ForEach applies fn to each bitmap in the fragment.
|
||||
func (tx *LMDBTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// ForEachRange applies fn on the selected range of bits on the chosen fragment.
|
||||
func (tx *LMDBTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Count operates on the full bitmap level, so it sums over all the containers
|
||||
// in the bitmap.
|
||||
func (tx *LMDBTx) Count(index, field, view string, shard uint64) (uint64, error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Max is the maximum bit-value in your bitmap.
|
||||
// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does.
|
||||
func (tx *LMDBTx) Max(index, field, view string, shard uint64) (uint64, error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// Min returns the smallest bit set in the fragment. If no bit is hot,
|
||||
// the second return argument is false.
|
||||
func (tx *LMDBTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// CountRange returns the count of hot bits in the start, end range on the fragment.
|
||||
// roaring.countRange counts the number of bits set between [start, end).
|
||||
func (tx *LMDBTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// OffsetRange creates a new roaring.Bitmap to return in other. For all the
|
||||
// hot bits in [start, endx) of the chosen fragment, it stores
|
||||
// them into other but with offset added to their bit position.
|
||||
// The primary client is doing this, using ShardWidth, already; see
|
||||
// fragment.rowFromStorage() in fragment.go. For example:
|
||||
//
|
||||
// data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard,
|
||||
// f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth)
|
||||
// ^ offset ^ start ^ endx
|
||||
//
|
||||
// The start and endx arguments are container keys that have been shifted left by 16 bits;
|
||||
// their highbits() will be taken to determine the actual container keys. This
|
||||
// is done to conform to the roaring.OffsetRange() argument convention.
|
||||
//
|
||||
func (tx *LMDBTx) OffsetRange(index, field, view string, shard, offset, start, endx uint64) (other *roaring.Bitmap, err error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// IncrementOpN increments the tx opcount by changedN
|
||||
func (tx *LMDBTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// ImportRoaringBits handles deletes by setting clear=true.
|
||||
// rowSet[rowID] returns the number of bit changed on that rowID.
|
||||
func (tx *LMDBTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (tx *LMDBTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// StringifiedLMDBKeys returns a string with all the container
|
||||
// keys available in lmdb.
|
||||
func (w *LMDBWrapper) StringifiedLMDBKeys(optionalUseThisTx Tx) (r string) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// countBitsSet returns the number of bits set (or "hot") in
|
||||
// the roaring container value found by the txkey.Key()
|
||||
// formatted bkey.
|
||||
func (tx *LMDBTx) countBitsSet(bkey []byte) (n int) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (tx *LMDBTx) IsDone() (done bool) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (tx *LMDBTx) Dump(short bool, shard uint64) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// stringifiedLMDBKeysTx reports all the lmdb keys and a
|
||||
// corresponding blake3 hash viewable by txn within the entire
|
||||
// lmdb database.
|
||||
// It also reports how many bits are hot in the roaring container
|
||||
// (how many bits are set, or 1 rather than 0).
|
||||
//
|
||||
// By convention, we must return the empty string if there
|
||||
// are no keys present. The tests use this to confirm
|
||||
// an empty database.
|
||||
func stringifiedLMDBKeysTx(tx *LMDBTx) (r string) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (w *LMDBWrapper) DeleteField(index, field, fieldPath string) error {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (w *LMDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (w *LMDBWrapper) DeletePrefix(prefix []byte) error {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (tx *LMDBTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
// UnionInPlace unions all the others Bitmaps into a new Bitmap, and then writes it to the
|
||||
// specified fragment.
|
||||
func (tx *LMDBTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
|
||||
func (tx *LMDBTx) Options() Txo {
|
||||
panic("lmdb only available on 64-bit arch")
|
||||
}
|
||||
1294
lmdb_test.go
1294
lmdb_test.go
File diff suppressed because it is too large
Load diff
32
txfactory.go
32
txfactory.go
|
|
@ -38,7 +38,6 @@ import (
|
|||
// public strings that pilosa/server/config.go can reference
|
||||
const (
|
||||
RoaringTxn string = "roaring"
|
||||
LmdbTxn string = "lmdb"
|
||||
RBFTxn string = "rbf"
|
||||
BoltTxn string = "bolt"
|
||||
)
|
||||
|
|
@ -53,7 +52,7 @@ const DefaultTxsrc = RBFTxn
|
|||
// which the transaction has committed or rolled back. Since
|
||||
// memory segments will be recycled by the underlying databases,
|
||||
// this can lead to corruption. When DetectMemAccessPastTx is true,
|
||||
// code in lmdb.go will copy the transactionally viewed memory before
|
||||
// code in bolt.go will copy the transactionally viewed memory before
|
||||
// returning it for bitmap reading, and then zero it or overwrite it
|
||||
// with -2 when the Tx completes.
|
||||
//
|
||||
|
|
@ -83,8 +82,7 @@ var sep = string(os.PathSeparator)
|
|||
// course that your "new" read Tx actually has an "old" view
|
||||
// of the database.
|
||||
//
|
||||
// At the moment, given that LMDB demands that
|
||||
// all write Tx are created and executed on the same C thread, most
|
||||
// At the moment, most
|
||||
// writes to individual shards are commited eagerly and locally
|
||||
// when the `defer finisher(&err0)` is run.
|
||||
// This is done by returning a finisher that actually Commits,
|
||||
|
|
@ -265,16 +263,8 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) {
|
|||
// don't deadlock against themselves under blue-green.
|
||||
o.Write = o.Write || qcx.write
|
||||
|
||||
// note: write Tx were re-using Tx across different goroutines,
|
||||
// which lmdb will not be pleased with. For reads this
|
||||
// should be okay, as the docs say
|
||||
// "If you want to pass read-only transactions across threads,
|
||||
// you can use the MDB_NOTLS option on the environment."
|
||||
// -- http://www.lmdb.tech/doc/starting.html
|
||||
// and we always use lmdb.NoTLS as the lmdb-go bindings ensure this.
|
||||
//
|
||||
// So we make ALL write transactions local, and never reuse them
|
||||
// below.
|
||||
// In general, we make ALL write transactions local, and never reuse them
|
||||
// below. Previously this was to help lmdb.
|
||||
//
|
||||
// *However* there is one exception: when we have set RequiredForAtomicWriteTx
|
||||
// for the importing of an AtomicRequest, then we must use that
|
||||
|
|
@ -400,7 +390,7 @@ func (qcx *Qcx) ListOpenTx() string {
|
|||
}
|
||||
|
||||
// TxFactory abstracts the creation of Tx interface-level
|
||||
// transactions so that RBF, BoltDB, LMDB, or Roaring-fragment-files, or several
|
||||
// transactions so that RBF, BoltDB, or Roaring-fragment-files, or several
|
||||
// of these at once in parallel, is used as the storage and transction layer.
|
||||
type TxFactory struct {
|
||||
typeOfTx string
|
||||
|
|
@ -435,13 +425,12 @@ const (
|
|||
noneTxn txtype = 0
|
||||
roaringTxn txtype = 1 // these don't really have any transactions
|
||||
rbfTxn txtype = 2
|
||||
lmdbTxn txtype = 3
|
||||
boltTxn txtype = 4
|
||||
)
|
||||
|
||||
// these need to be skipped by the holder.go field scanner that
|
||||
// calls IsTxDatabasePath
|
||||
var allTypesWithSuffixes = []txtype{rbfTxn, lmdbTxn, boltTxn}
|
||||
var allTypesWithSuffixes = []txtype{rbfTxn, boltTxn}
|
||||
|
||||
// FileSuffix is used to determine backend directory names.
|
||||
// We append '@' to be sure we never collide with a field name
|
||||
|
|
@ -454,8 +443,6 @@ func (ty txtype) FileSuffix() string {
|
|||
return ""
|
||||
case rbfTxn:
|
||||
return "-rbfdb@"
|
||||
case lmdbTxn:
|
||||
return "-lmdb@"
|
||||
case boltTxn:
|
||||
return "-boltdb@"
|
||||
}
|
||||
|
|
@ -504,8 +491,6 @@ func MustTxsrcToTxtype(txsrc string) (types []txtype) {
|
|||
types = append(types, roaringTxn)
|
||||
case RBFTxn: // "rbf"
|
||||
types = append(types, rbfTxn)
|
||||
case LmdbTxn: // "lmdb"
|
||||
types = append(types, lmdbTxn)
|
||||
case BoltTxn: // "bolt"
|
||||
types = append(types, boltTxn)
|
||||
default:
|
||||
|
|
@ -886,8 +871,6 @@ func (ty txtype) String() string {
|
|||
return "roaring"
|
||||
case rbfTxn:
|
||||
return "rbf"
|
||||
case lmdbTxn:
|
||||
return "lmdb"
|
||||
case boltTxn:
|
||||
return "bolt"
|
||||
}
|
||||
|
|
@ -1253,9 +1236,6 @@ func anyGlobalDBWrappersStillOpen() bool {
|
|||
if globalRbfDBReg.Size() != 0 {
|
||||
return true
|
||||
}
|
||||
if globalLMDBReg.Size() != 0 {
|
||||
return true
|
||||
}
|
||||
if globalBoltReg.Size() != 0 {
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/glycerine/lmdb-go/lmdb"
|
||||
//"github.com/pilosa/pilosa/v2/logger"
|
||||
)
|
||||
|
||||
func Test_TxFactory_Qcx_query_context(t *testing.T) {
|
||||
|
|
@ -121,7 +120,7 @@ func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) {
|
|||
orig := os.Getenv("PILOSA_TXSRC")
|
||||
defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
|
||||
|
||||
checked := []string{"lmdb", "roaring", "rbf"}
|
||||
checked := []string{"roaring", "rbf"}
|
||||
|
||||
expectError := false
|
||||
for _, blue := range checked {
|
||||
|
|
@ -270,7 +269,7 @@ func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) {
|
|||
orig := os.Getenv("PILOSA_TXSRC")
|
||||
defer os.Setenv("PILOSA_TXSRC", orig) // must restore or will mess up other tests!
|
||||
|
||||
checked := []string{"lmdb", "roaring", "bolt", "rbf"}
|
||||
checked := []string{"roaring", "bolt", "rbf"}
|
||||
|
||||
for _, blue := range checked {
|
||||
for _, green := range checked {
|
||||
|
|
@ -408,8 +407,8 @@ func Test_TxFactory_verifyStringConstantsMatch(t *testing.T) {
|
|||
// our const definitions at the top of txfactory.go, or
|
||||
// else blue-green transactions cannot determine when
|
||||
// the second transaction is being released in dbshard.go.
|
||||
check := []txtype{roaringTxn, rbfTxn, lmdbTxn, boltTxn}
|
||||
expect := []string{RoaringTxn, RBFTxn, LmdbTxn, BoltTxn}
|
||||
check := []txtype{roaringTxn, rbfTxn, boltTxn}
|
||||
expect := []string{RoaringTxn, RBFTxn, BoltTxn}
|
||||
for i, chk := range check {
|
||||
obs := chk.String()
|
||||
if obs != expect[i] {
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ func init() {
|
|||
// keeper linter happy
|
||||
_ = pp
|
||||
_ = vv
|
||||
_ = DirExists
|
||||
}
|
||||
|
||||
func PanicOn(err error) {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue