scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 09:05:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #659 from molecula/rbf_99pct

Browse source 
all test green on rbf. WOOT.
This commit is contained in:
jaten-molecula 2020-08-12 16:13:47 -05:00 • committed by GitHub
commit 973e9a7955
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
60 changed files with 7151 additions and 1946 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
Makefile
View file

@ -172,6 +172,18 @@ topt-rbf:
@echo " log.topt.rbf green: \c"; cat log.topt.rbf | grep PASS |wc -l
@echo " log.topt.rbf red: \c"; cat log.topt.rbf | grep '\-\-\- FAIL' |wc -l
topt-rbf-race:
mv log.topt.rbf-race log.topt.rbf-race.prev || true
PILOSA_TXSRC=rbf go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.rbf-race
@echo " log.topt.rbf-race green: \c"; cat log.topt.rbf-race | grep PASS |wc -l
@echo " log.topt.rbf-race red: \c"; cat log.topt.rbf-race | grep '\-\-\- FAIL' |wc -l
topt-lmdb:
mv log.topt.lmdb log.topt.lmdb.prev || true
PILOSA_TXSRC=lmdb go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lmdb
@echo " log.topt.lmdb green: \c"; cat log.topt.lmdb | grep PASS |wc -l
@echo " log.topt.lmdb red: \c"; cat log.topt.lmdb | grep '\-\-\- FAIL' |wc -l
topt-race:
mv log.topt.race log.topt.race.prev || true
go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.race
@ -221,7 +233,7 @@ bg-rbf:
# Run golangci-lint
golangci-lint: require-golangci-lint
golangci-lint run --skip-files '.*\.peg\.go'
golangci-lint run --timeout 3m --skip-files '.*\.peg\.go'
# Alias
linter: golangci-lint

17
api.go
View file

@ -1769,6 +1769,23 @@ func (api *API) ActiveQueries(ctx context.Context) ([]ActiveQueryStatus, error)
return api.tracker.ActiveQueries(), nil
}
// TranslateIndexDB is an internal function to load the index keys database
func (api *API) TranslateIndexDB(ctx context.Context, indexName string, partitionID int, rd io.Reader) error {
idx := api.holder.Index(indexName)
store := idx.TranslateStore(partitionID)
_, err := store.ReadFrom(rd)
return err
}
// TranslateFieldDB is an internal function to load the field keys database
func (api *API) TranslateFieldDB(ctx context.Context, indexName, fieldName string, rd io.Reader) error {
idx := api.holder.Index(indexName)
field := idx.Field(fieldName)
store := field.TranslateStore()
_, err := store.ReadFrom(rd)
return err
}
type serverInfo struct {
ShardWidth uint64 `json:"shardWidth"`
Memory uint64 `json:"memory"`

5
api_test.go
View file

@ -134,8 +134,9 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(res.ColumnAttrSets) != 100 {
t.Fatal("incorrect number of column attrs set")
m := len(res.ColumnAttrSets)
if m != 100 {
t.Fatalf("incorrect number of column attrs set; m = %v", m)
}
for _, v := range res.ColumnAttrSets {

317
badger.go
View file

@ -23,7 +23,6 @@ import (
"os"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"time"
@ -32,6 +31,7 @@ import (
badger "github.com/dgraph-io/badger/v2"
badgeroptions "github.com/dgraph-io/badger/v2/options"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txpath"
"github.com/pkg/errors"
)
@ -113,12 +113,19 @@ import (
var badgerDefaultLogger *BadgerLog
var badgerTestLogger *BadgerLog
const BadgerLogToStderr = false
func init() {
// badger test output clutters up the screen, dump to /dev/null for now.
// TODO(jea): figure out where badger logging should go.
null, err := os.Open(os.DevNull)
panicOn(err)
badgerTestLogger = &BadgerLog{Logger: log.New(null, "badger ", log.LstdFlags)}
var out io.Writer = null
if BadgerLogToStderr {
// view badger logs
out = os.Stderr
}
badgerTestLogger = &BadgerLog{Logger: log.New(out, "badger ", log.LstdFlags)}
badgerDefaultLogger = badgerTestLogger
// BadgerDB recommends a minimum of 128 GOMAXPROCS to make use of the IOPs
@ -247,17 +254,6 @@ func DumpAllBadger() {
}
}
// newBadgerDBWrapper creates a new empty database, blowing away
// any prior path + "-badgerdb" directory.
func (r *badgerRegistrar) newBadgerDBWrapper(path string) (*BadgerDBWrapper, error) {
bpath := badgerPath(path)
err := os.RemoveAll(bpath)
if err != nil {
return nil, err
}
return r.openBadgerDBWrapper(bpath)
}
// badgerPath is a helper for determining the full directory
// in which the badger database will be stored.
func badgerPath(path string) string {
@ -297,6 +293,7 @@ func (r *badgerRegistrar) openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, e
opt.Compression = badgeroptions.None // turn off compression.
opt.ZSTDCompressionLevel = 0 // really, just in case.
opt.SyncWrites = true // default is true, safe.
//opt.KeepL0InMemory = true // speedup?
// MaxCacheSize docs:
//
@ -307,9 +304,15 @@ func (r *badgerRegistrar) openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, e
// encryption both are disabled, adding a cache will lead to
// unnecessary overhead which will affect the read performance.
// Setting size to zero disables the cache altogether.
//opt.MaxCacheSize = 1 << 30 // slows down 135 sec vs 113 sec on our benchmark
opt.MaxCacheSize = 0
opt.LoadBloomsOnOpen = false // should speed up start-up time.
//opt.KeepBlocksInCache = true // default false
//opt.KeepBlockIndicesInCache = true // default false
opt.BlockSize = 8 * 1024 // default 4 * 1024
// to get memory only do:
//opt := badger.DefaultOptions("").WithLogger(badgerDefaultLogger).WithInMemory(true)
@ -342,7 +345,7 @@ func (w *BadgerDBWrapper) DeleteIndex(indexName string) error {
if strings.Contains(indexName, "'") {
return fmt.Errorf("error: bad indexName `%v` in BadgerDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes.", indexName)
}
prefix := badgerIndexOnlyPrefix(indexName)
prefix := txpath.IndexOnlyPrefix(indexName)
return w.DeletePrefix(prefix)
}
@ -475,27 +478,19 @@ func (w *BadgerDBWrapper) UnprotectedListOpenItAsString() (r string) {
// but when set is highly useful for debugging. It has no impact
// on transaction behavior.
//
func (w *BadgerDBWrapper) NewBadgerTx(write bool, initialIndexName string) (tx *BadgerTx) {
w.muDb.Lock()
defer w.muDb.Unlock()
func (w *BadgerDBWrapper) NewBadgerTx(write bool, initialIndexName string, frag *fragment) (tx *BadgerTx) {
tx = &BadgerTx{
frag: frag,
write: write,
tx: w.db.NewTransaction(write),
Db: w,
initloc: stack(),
doAllocZero: w.doAllocZero,
initialIndexName: initialIndexName,
DeleteEmptyContainer: w.DeleteEmptyContainer,
//initloc: "", // stack(),
}
if w.openTx == nil {
w.openTx = make(map[*BadgerTx]bool)
}
w.muOpenTxIt.Lock()
w.openTx[tx] = write
w.muOpenTxIt.Unlock()
return
}
@ -531,9 +526,11 @@ type BadgerTx struct {
Db *BadgerDBWrapper
tx *badger.Txn
frag *fragment
opcount int
initloc string // stack trace of where we were initially created.
// keep linter happy, comment out until needed again for debugging.
//initloc string // stack trace of where we were initially created.
doAllocZero bool
@ -554,7 +551,8 @@ func (tx *BadgerTx) Type() string {
}
func (tx *BadgerTx) UseRowCache() bool {
return false
//the row cache speeds up queries.
return true
}
// overWriteOurAllocs provides detection of memory
@ -592,12 +590,6 @@ func (tx *BadgerTx) overWriteOurAllocs() {
//}
}
// WholeDatabaseBlake3Hash returns the root-hash from the Merkle tree
// built by hashing all bits stored in the database backing this transaction.
func (tx *BadgerTx) WholeDatabaseBlake3Hash(index, field, view string, shard uint64) (hash string, err error) {
return
}
// Pointer gives us a memory address for the underlying transaction for debugging.
// It is public because we use it in roaring to report invalid container memory access
// outside of a transaction.
@ -666,122 +658,6 @@ func (tx *BadgerTx) RoaringBitmap(index, field, view string, shard uint64) (*roa
return tx.OffsetRange(index, field, view, shard, 0, 0, LeftShifted16MaxContainerKey)
}
// badgerKey produces the bytes that we use as a key to query badger.
// The roaringContainerKey argument is a container key into a roaring Container.
// Output examples:
//
// "idx:'i';fld:'f';vw:'standard';shd:'0';ckey@00000000000000000000" // smallest container-key
// "idx:'i';fld:'f';vw:'standard';shd:'0';ckey@18446744073709551615" // largest container-key (math.MaxUint64)
//
// NB must be kept in sync with badgerPrefix() and badgerKeyExtractContainerKey().
//
func badgerKey(index, field, view string, shard uint64, roaringContainerKey uint64) []byte {
// The %020d which adds zero padding up to 20 runes is required to
// allow the textual sort to accurately
// reflect a numeric sort order. This is because, as a string,
// math.MaxUint64 is 20 bytes long.
// Example of such a badgerKey with a container-key that is math.MaxUint64:
// ...........................................12345678901234567890
// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615
prefix := badgerPrefix(index, field, view, shard)
ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
bkey := append(prefix, ckey...)
MustValidateKey(bkey)
return bkey
}
var ckeyPartExpected = []byte(";ckey@")
// MustValidatekey will panic on a bad badgerKey with an informative message.
func MustValidateKey(bkey []byte) {
n := len(bkey)
if n < 56 {
panic(fmt.Sprintf("bkey too short min size is 56 but we see %v in '%v'", n, string(bkey)))
}
beforeCkey := bkey[n-26 : n-20]
if !bytes.Equal(beforeCkey, ckeyPartExpected) {
panic(fmt.Sprintf(`bkey did not have expected ";ckey@" at 26 bytes from the end of the bkey '%v'; instead had '%v'`, string(bkey), string(beforeCkey)))
}
}
func shardFromBadgerKey(bkey []byte) (shard uint64) {
MustValidateKey(bkey)
n := len(bkey)
// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615 -> idx:'i';fld:'f';vw:'standard';shd:'1
by := bkey[:n-27]
beg := bytes.LastIndex(by, []byte("'"))
if beg == -1 {
panic(fmt.Sprintf("bad bkey='%v' did not have single quote to being shard decoding", string(bkey)))
}
parseMe := string(by[beg+1:])
shard, err := strconv.ParseUint(parseMe, 10, 64)
if err != nil {
panic(fmt.Sprintf("could not parse parseMe '%v' in strconv.ParseUint(), error: '%v'", parseMe, err))
}
return shard
}
// badgerKeyAndPrefix returns the equivalent of badgerKey() and badgerPrefix() calls.
func badgerKeyAndPrefix(index, field, view string, shard uint64, roaringContainerKey uint64) (key, prefix []byte) {
prefix = badgerPrefix(index, field, view, shard)
ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
bkey := append(prefix, ckey...)
MustValidateKey(bkey)
return bkey, prefix
}
var _ = badgerKeyAndPrefix // keep linter happy
// badgerKeyExtractContainerKey extracts the containerKey from bkey.
func badgerKeyExtractContainerKey(bkey []byte) (containerKey uint64) {
MustValidateKey(bkey)
// The zero padding means that the container-key is always the last 20 bytes of the bkey.
//
// Be sure to catch the problematic case of a user passing in only a prefix. A prefix
// ends in 'key@' rather than a full key that has 'key@00000000000000000001' (for example)
// at the end. The ParseUint call below will fail in that case.
n := len(bkey)
if n < 20 {
panic(fmt.Sprintf("badgerKeyExtractContainerKey() error: bad bkey '%v', too short!", string(bkey)))
}
last := bkey[n-20:] // badgerKey() and badgerPrefix() always return more than 20 rune []byte.
var err error
containerKey, err = strconv.ParseUint(string(last), 10, 64) // has to be the container key
if err != nil {
panic(fmt.Sprintf("badgerKeyExtractContainerKey() error: bad bkey '%v', could not convert last 20 bytes ('%v') to a unit64: '%v'", string(bkey), string(last), err))
}
return
}
func badgerAllShardPrefix(index, field, view string) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:", index, field, view))
}
// badgerPrefix returns everything from badgerKey up to and
// including the '@' fune in a badger key. The prefix excludes the roaring container key itself.
// NB must be kept in sync with badgerKey() and badgerKeyExtractContainerKey().
func badgerPrefix(index, field, view string, shard uint64) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@", index, field, view, shard))
}
// badgerIndexOnlyPrefix returns a prefix suitable for DeleteIndex and a key-scan to
// remove all storage associated with one index.
//
// The full name of the index must be provided, no partial index names will work.
//
// The provided key is terminated by `';` and so DeleteIndex("i") will not delete the index "i2".
//
func badgerIndexOnlyPrefix(indexName string) []byte {
return []byte(fmt.Sprintf("idx:'%v';", indexName))
}
// same for deleting a whole field.
func badgerFieldPrefix(index, field string) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';", index, field))
}
// Container returns the requested roaring.Container, selected by fragment and ckey
func (tx *BadgerTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) {
@ -790,7 +666,7 @@ func (tx *BadgerTx) Container(index, field, view string, shard uint64, ckey uint
// you must use copy() to copy it to another byte slice.
// BUT here we are already inside the Txn.
bkey := badgerKey(index, field, view, shard, ckey)
bkey := txpath.Key(index, field, view, shard, ckey)
tx.mu.Lock()
var item *badger.Item
item, err = tx.tx.Get(bkey)
@ -815,22 +691,22 @@ func (tx *BadgerTx) Container(index, field, view string, shard uint64, ckey uint
// PutContainer stores rc under the specified fragment and container ckey.
func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey uint64, rc *roaring.Container) error {
bkey := badgerKey(index, field, view, shard, ckey)
bkey := txpath.Key(index, field, view, shard, ckey)
var by []byte
ct := roaring.ContainerType(rc)
switch ct {
case containerArray:
case roaring.ContainerArray:
by = fromArray16(roaring.AsArray(rc))
case containerBitmap:
case roaring.ContainerBitmap:
by = fromArray64(roaring.AsBitmap(rc))
case containerRun:
case roaring.ContainerRun:
by = fromInterval16(roaring.AsRuns(rc))
case containerNil:
panic("wat? nil container is unexpected, no?!?")
case roaring.ContainerNil:
panic("wat? nil roaring.Container is unexpected, no?!?")
default:
panic(fmt.Sprintf("unknown container type: %v", ct))
panic(fmt.Sprintf("unknown roaring.Container type: %v", ct))
}
entry := badger.NewEntry(bkey, by).WithMeta(ct)
tx.mu.Lock()
@ -857,7 +733,7 @@ func (tx *BadgerTx) PutContainer(index, field, view string, shard uint64, ckey u
// RemoveContainer deletes the container specified by the shard and container key ckey
func (tx *BadgerTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error {
bkey := badgerKey(index, field, view, shard, ckey)
bkey := txpath.Key(index, field, view, shard, ckey)
tx.mu.Lock()
err := tx.tx.Delete(bkey)
tx.mu.Unlock()
@ -949,7 +825,7 @@ func (tx *BadgerTx) Remove(index, field, view string, shard uint64, a ...uint64)
func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
lo, hi := lowbits(key), highbits(key)
bkey := badgerKey(index, field, view, shard, hi)
bkey := txpath.Key(index, field, view, shard, hi)
tx.mu.Lock()
item, err := tx.tx.Get(bkey)
tx.mu.Unlock()
@ -970,7 +846,7 @@ func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64
func (tx *BadgerTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
prefix := badgerAllShardPrefix(index, field, view)
prefix := txpath.AllShardPrefix(index, field, view)
bi := NewBadgerIterator(tx, prefix)
defer bi.Close()
@ -983,7 +859,7 @@ func (tx *BadgerTx) SliceOfShards(index, field, view, optionalViewPath string) (
for bi.Next() {
item := bi.it.Item()
key := item.Key()
shard := shardFromBadgerKey(key)
shard := txpath.ShardFromKey(key)
if firstDone {
if shard != lastShard {
sliceOfShards = append(sliceOfShards, shard)
@ -1009,10 +885,10 @@ func (tx *BadgerTx) SliceOfShards(index, field, view, optionalViewPath string) (
func (tx *BadgerTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
// needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000"
needle := badgerKey(index, field, view, shard, firstRoaringContainerKey)
needle := txpath.Key(index, field, view, shard, firstRoaringContainerKey)
// prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@"
prefix := badgerPrefix(index, field, view, shard)
prefix := txpath.Prefix(index, field, view, shard)
bi := NewBadgerIterator(tx, prefix)
bi.Seek(needle)
@ -1045,12 +921,9 @@ type BadgerIterator struct {
}
// NewBadgerIterator creates an iterator on tx that will
// only return badgerKeys that start with prefix.
// only return txpath.Keys that start with prefix.
func NewBadgerIterator(tx *BadgerTx, prefix []byte) (bi *BadgerIterator) {
tx.Db.muOpenTxIt.Lock()
defer tx.Db.muOpenTxIt.Unlock()
opts := badger.DefaultIteratorOptions
opts.PrefetchValues = false // else by default, pre-fetches the 1st 100 values, which would be slow.
opts.Reverse = false
@ -1064,10 +937,13 @@ func NewBadgerIterator(tx *BadgerTx, prefix []byte) (bi *BadgerIterator) {
it: it,
prefix: prefix,
}
tx.Db.muOpenTxIt.Lock()
if tx.Db.openIt == nil {
tx.Db.openIt = make(map[*BadgerIterator]bool)
}
tx.Db.openIt[bi] = false // true for reverse, false for forward iteration.
tx.Db.muOpenTxIt.Unlock()
bi.it.Seek(prefix)
return
}
@ -1150,7 +1026,7 @@ func (bi *BadgerIterator) Value() (containerKey uint64, c *roaring.Container) {
panic("item was nil")
}
key := item.Key()
containerKey = badgerKeyExtractContainerKey(key)
containerKey = txpath.KeyExtractContainerKey(key)
err := item.Value(func(v []byte) error {
c = bi.tx.toContainer(item.UserMeta(), v)
@ -1256,8 +1132,8 @@ func (tx *BadgerTx) Count(index, field, view string, shard uint64) (uint64, erro
// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does.
func (tx *BadgerTx) Max(index, field, view string, shard uint64) (uint64, error) {
prefix := badgerPrefix(index, field, view, shard)
seekto := badgerPrefix(index, field, view, shard+1)
prefix := txpath.Prefix(index, field, view, shard)
seekto := txpath.Prefix(index, field, view, shard+1)
it := NewBadgerReverseIterator(tx, prefix, seekto) // this iterator is still open, when we commit/discard tx.
defer it.Close()
@ -1318,6 +1194,24 @@ func (tx *BadgerTx) UnionInPlace(index, field, view string, shard uint64, others
// roaring.countRange counts the number of bits set between [start, end).
func (tx *BadgerTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
if tx.frag == nil {
return tx.countRangeNoFrag(index, field, view, shard, start, end)
}
// For speed, exploit the fact that on startup the rowCache will
// have already loaded fragments.
rowID := start / ShardWidth
row, err := tx.frag.unprotectedRow(tx, rowID)
if err != nil {
return 0, err
}
return row.Count(), nil
}
// CountRange returns the count of hot bits in the start, end range on the fragment.
// roaring.countRange counts the number of bits set between [start, end).
func (tx *BadgerTx) countRangeNoFrag(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
if start >= end {
return 0, nil
}
@ -1398,27 +1292,24 @@ func (tx *BadgerTx) OffsetRange(index, field, view string, shard, offset, start,
off := highbits(offset)
hi0, hi1 := highbits(start), highbits(endx)
// TODO(jea): question: do we have to account for ShardWidth here? what if the move goes
// beyond a shard?
needle := txpath.Key(index, field, view, shard, hi0)
prefix := txpath.Prefix(index, field, view, shard)
needle := badgerKey(index, field, view, shard, hi0)
prefix := badgerPrefix(index, field, view, shard)
n2, pre2 := badgerKeyAndPrefix(index, field, view, shard, hi0)
n2, pre2 := txpath.KeyAndPrefix(index, field, view, shard, hi0)
if string(n2) != string(needle) {
panic(fmt.Sprintf("problem! n2(%v) != needle(%v), badgerKeyAndPrefix not consitent with badgerKey()", string(n2), string(needle)))
panic(fmt.Sprintf("problem! n2(%v) != needle(%v), txpath.KeyAndPrefix not consitent with txpath.Key()", string(n2), string(needle)))
}
if string(pre2) != string(prefix) {
panic(fmt.Sprintf("problem! pre2(%v) != prefix(%v), badgerKeyAndPrefix not consitent with badgerKey()", string(pre2), string(prefix)))
panic(fmt.Sprintf("problem! pre2(%v) != prefix(%v), txpath.KeyAndPrefix not consitent with txpath.Key()", string(pre2), string(prefix)))
}
it := NewBadgerIterator(tx, prefix) // see OffsetRange() panic 'Only one iterator can be active at one time, for a RW txn
it := NewBadgerIterator(tx, prefix)
defer it.Close()
it.Seek(needle)
for ; it.it.ValidForPrefix(prefix); it.Next() {
item := it.it.Item()
bkey := item.Key()
k := badgerKeyExtractContainerKey(bkey)
k := txpath.KeyExtractContainerKey(bkey)
// >= hi1 is correct b/c endx cannot have any lowbits set.
if uint64(k) >= hi1 {
@ -1426,7 +1317,6 @@ func (tx *BadgerTx) OffsetRange(index, field, view string, shard, offset, start,
}
destCkey := off + (k - hi0)
err := item.Value(func(v []byte) error {
c := tx.toContainer(item.UserMeta(), v)
other.Containers.Put(destCkey, c.Freeze())
@ -1538,7 +1428,7 @@ func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, i
newC := roaring.Union(oldC, synthC) // UnionInPlace was giving us crashes on overly large containers.
if roaring.ContainerType(newC) == containerBitmap {
if roaring.ContainerType(newC) == roaring.ContainerBitmap {
newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it.
}
if newC.N() != existN {
@ -1574,14 +1464,6 @@ func toInterval16(a []byte) []roaring.Interval16 {
return (*[2048]roaring.Interval16)(unsafe.Pointer(&a[0]))[: len(a)/4 : len(a)/4]
}
// should really be exported from the pilosa/roaring package so we don't get out of sync...
const (
containerNil byte = iota // no container
containerArray // slice of bit position values
containerBitmap // slice of 1024 uint64s
containerRun // container of run-encoded bits
)
func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
if len(v) == 0 {
@ -1589,7 +1471,8 @@ func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
}
var w []byte
if tx.doAllocZero {
useRowCache := tx.UseRowCache()
if tx.doAllocZero || useRowCache {
// Do electric fence-inspired bad-memory read detection.
//
// The v []byte lives in BadgerDB's memory-mapped vlog-file,
@ -1610,26 +1493,37 @@ func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
w = make([]byte, len(v))
copy(w, v)
// register w so we can catch out-of-tx memory access
tx.acMu.Lock()
defer tx.acMu.Unlock()
tx.ourAllocs = append(tx.ourAllocs, w)
if !useRowCache {
// register w so we can catch out-of-tx memory access
tx.acMu.Lock()
defer tx.acMu.Unlock()
tx.ourAllocs = append(tx.ourAllocs, w)
}
} else {
w = v
}
switch typ {
case containerArray:
case roaring.ContainerArray:
c := roaring.NewContainerArray(toArray16(w))
tx.ourContainers = append(tx.ourContainers, c)
if tx.doAllocZero {
// tx.acMu was acquired above, and Unlock deferred.
tx.ourContainers = append(tx.ourContainers, c)
}
return c
case containerBitmap:
case roaring.ContainerBitmap:
c := roaring.NewContainerBitmap(-1, toArray64(w))
tx.ourContainers = append(tx.ourContainers, c)
if tx.doAllocZero {
// tx.acMu was acquired above, and Unlock deferred.
tx.ourContainers = append(tx.ourContainers, c)
}
return c
case containerRun:
case roaring.ContainerRun:
c := roaring.NewContainerRun(toInterval16(w))
tx.ourContainers = append(tx.ourContainers, c)
if tx.doAllocZero {
// tx.acMu was acquired above, and Unlock deferred.
tx.ourContainers = append(tx.ourContainers, c)
}
return c
default:
panic(fmt.Sprintf("unknown container: %v", typ))
@ -1670,7 +1564,7 @@ func fromInterval16(a []roaring.Interval16) []byte {
// keys available in badger.
func (w *BadgerDBWrapper) StringifiedBadgerKeys(optionalUseThisTx Tx) (r string) {
if optionalUseThisTx == nil {
tx := w.NewBadgerTx(!writable, "<StringifiedBadgerKeys>")
tx := w.NewBadgerTx(!writable, "<StringifiedBadgerKeys>", nil)
defer tx.Rollback()
r = stringifiedBadgerKeysTx(tx)
return
@ -1685,7 +1579,7 @@ func (w *BadgerDBWrapper) StringifiedBadgerKeys(optionalUseThisTx Tx) (r string)
}
// countBitsSet returns the number of bits set (or "hot") in
// the roaring container value found by the badgerKey()
// the roaring container value found by the txpath.Key()
// formatted bkey.
func (tx *BadgerTx) countBitsSet(bkey []byte) (n int) {
@ -1723,7 +1617,7 @@ func (tx *BadgerTx) Dump() {
func stringifiedBadgerKeysTx(tx *BadgerTx) (r string) {
r = "allkeys:[\n"
it := tx.tx.NewIterator(badger.DefaultIteratorOptions)
it := tx.tx.NewIterator(badger.DefaultIteratorOptions) // PrefetchValues true okay here.
defer it.Close()
any := false
for it.Rewind(); it.Valid(); it.Next() {
@ -1731,7 +1625,7 @@ func stringifiedBadgerKeysTx(tx *BadgerTx) (r string) {
item := it.Item()
bkey := item.Key()
key := string(bkey)
ckey := badgerKeyExtractContainerKey(bkey)
ckey := txpath.KeyExtractContainerKey(bkey)
hash := ""
srbm := ""
err := item.Value(func(val []byte) error {
@ -1794,9 +1688,9 @@ func zeroKeyContainerAsString(ct *roaring.Container) (r string) {
}
var containerTypeNames = map[byte]string{
containerArray: "array",
containerBitmap: "bitmap",
containerRun: "run",
roaring.ContainerArray: "array",
roaring.ContainerBitmap: "bitmap",
roaring.ContainerRun: "run",
}
func bitmapAsString(rbm *roaring.Bitmap) (r string) {
@ -1889,12 +1783,12 @@ func (w *BadgerDBWrapper) DeleteField(index, field, fieldPath string) error {
if err != nil {
return errors.Wrap(err, "removing directory")
}
prefix := badgerFieldPrefix(index, field)
prefix := txpath.FieldPrefix(index, field)
return w.DeletePrefix(prefix)
}
func (w *BadgerDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
prefix := badgerPrefix(index, field, view, shard)
prefix := txpath.Prefix(index, field, view, shard)
return w.DeletePrefix(prefix)
}
@ -1922,7 +1816,8 @@ func (w *BadgerDBWrapper) DeletePrefix(prefix []byte) error {
o.PrefetchValues = false // key-only iteration, no values.
// note: panic: Unclosed iterator at time of Txn.Discard ? panic on segfault here?
// This means we messed up and Closed() the Database already; too early.
// This means we messed up and Closed() the Database already; too early. For
// example in TxFactor.CloseIndex() in txfactory.go:331.
it := txn.NewIterator(o)
defer it.Close()

420
badger_test.go
View file

@ -32,7 +32,6 @@ import (
"fmt"
"math"
"os"
"strconv"
"testing"
"github.com/dgraph-io/badger/v2"
@ -46,7 +45,7 @@ var _ = &roaring.Bitmap{}
func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) {
tx := dbwrap.NewBadgerTx(!writable, index)
tx := dbwrap.NewBadgerTx(!writable, index, nil)
defer tx.Rollback()
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
@ -59,7 +58,7 @@ func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string
func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) {
tx := dbwrap.NewBadgerTx(!writable, index)
tx := dbwrap.NewBadgerTx(!writable, index, nil)
defer tx.Rollback()
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
@ -70,7 +69,7 @@ func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view str
}
func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
// add a bit
changed, err := tx.Add(index, field, view, shard, doBatched, putme)
@ -88,14 +87,14 @@ func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string,
}
func badgerDBMustDeleteBitvalueContainer(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
hi := highbits(putme)
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
panicOn(tx.Commit())
}
func badgerDBMustDeleteBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
_, err := tx.Remove(index, field, view, shard, putme)
panicOn(err)
panicOn(tx.Commit())
@ -105,7 +104,7 @@ func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func()
var err error
fn := badgerPath(path)
panicOn(os.RemoveAll(fn))
w, err = globalBadgerReg.newBadgerDBWrapper(path)
w, err = globalBadgerReg.openBadgerDBWrapper(path)
panicOn(err)
// verify it is empty
@ -120,13 +119,107 @@ func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func()
}
}
//
// end of helper utilities
//////////////////////////
//////////////////////////
// begin Tx method tests
func TestBadger_DeleteFragment(t *testing.T) {
// setup
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteFragment")
defer clean()
defer dbwrap.Close()
index, field, view, shard0 := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index, nil)
shard1 := uint64(1)
bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16}
shards := []uint64{shard0, shard1}
for _, s := range shards {
for _, v := range bits {
changed, err := tx.Add(index, field, view, s, doBatched, v)
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
}
}
for _, s := range shards {
for _, v := range bits {
exists, err := tx.Contains(index, field, view, s, v)
panicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!!")
}
}
}
err := tx.Commit()
panicOn(err)
// end of setup
survivor := shard0
victim := shard1
err = dbwrap.DeleteFragment(index, field, view, victim, nil)
panicOn(err)
tx = dbwrap.NewBadgerTx(!writable, index, nil)
defer tx.Rollback()
for _, s := range shards {
for _, v := range bits {
exists, err := tx.Contains(index, field, view, s, v)
panicOn(err)
if s == survivor {
if !exists {
panic(fmt.Sprintf("ARG survivor died : bit %v", v))
}
} else if s == victim { // victim, should have been deleted
if exists {
panic(fmt.Sprintf("ARG victim lived : bit %v", v))
}
}
}
}
}
func TestBadger_Max_on_many_containers(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Max_on_many_containers")
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
putmeValues := []uint64{0, 2 << 16, 4 << 16}
for _, putme := range putmeValues {
badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme)
badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable, index, nil)
defer tx.Rollback()
max, err := tx.Max(index, field, view, shard)
panicOn(err)
expected := putmeValues[len(putmeValues)-1]
if max != expected {
panic(fmt.Sprintf("expected Max() of %v but got max=%v", expected, max))
}
}
// and the rest
func TestBadger_SetBitmap(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SetBitmap")
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
bitvalue := uint64(0)
changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
if changed <= 0 {
@ -147,7 +240,7 @@ func TestBadger_SetBitmap(t *testing.T) {
// commited, so should be visible outside the txn
//
tx2 := dbwrap.NewBadgerTx(!writable, index)
tx2 := dbwrap.NewBadgerTx(!writable, index, nil)
exists, err = tx2.Contains(index, field, view, shard, bitvalue)
panicOn(err)
if !exists {
@ -163,11 +256,11 @@ func TestBadger_SetBitmap(t *testing.T) {
}
func TestBadger_OffsetRange(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SetBitmap")
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_OffsetRange")
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
bitvalue := uint64(1 << 20)
changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
@ -201,7 +294,7 @@ func TestBadger_OffsetRange(t *testing.T) {
start := uint64(0 << 16)
endx := bitvalue + 1<<16
tx2 := dbwrap.NewBadgerTx(!writable, index)
tx2 := dbwrap.NewBadgerTx(!writable, index, nil)
rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
tx2.Rollback()
@ -215,7 +308,7 @@ func TestBadger_OffsetRange(t *testing.T) {
// now offset by 2M
offset = uint64(2 << 20)
tx3 := dbwrap.NewBadgerTx(!writable, index)
tx3 := dbwrap.NewBadgerTx(!writable, index, nil)
rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
tx3.Rollback()
@ -243,7 +336,7 @@ func TestBadger_Count_on_many_containers(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
defer tx.Rollback()
n, err := tx.Count(index, field, view, shard)
@ -259,7 +352,7 @@ func TestBadger_Count_dense_containers(t *testing.T) {
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
expected := 0
// can't do more than about 100k writes per badger txn by default, so
@ -288,7 +381,7 @@ func TestBadger_ContainerIterator_on_empty(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(!writable, index)
tx := dbwrap.NewBadgerTx(!writable, index, nil)
defer tx.Rollback()
bitvalue := uint64(0)
citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue)
@ -306,7 +399,7 @@ func TestBadger_ContainerIterator_on_one_bit(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
defer tx.Rollback()
bitvalue := uint64(42)
@ -358,55 +451,12 @@ func TestBadger_ContainerIterator_on_one_bit(t *testing.T) {
}
}
func TestBadger_badgerKey_badgerPrefix(t *testing.T) {
// badgerPrefix() must agree with badgerKey(), but not have the key at the end.
// This is important for iteration over containers.
index, field, view, shard := "i", "f", "v", uint64(0)
// needle examples with the container-key extremes:
// "index:'i';field:'f';view:'v';shard:'0';key@00000000000000000000" // smallest
// "index:'i';field:'f';view:'v';shard:'0';key@18446744073709551615" // largest
needle := badgerKey(index, field, view, shard, 0)
// prefix example: "index:'i';field:'f';view:'v';shard:'0';key@"
prefix := badgerPrefix(index, field, view, shard)
if !bytes.HasPrefix(needle, prefix) {
panic(fmt.Sprintf("badgerPrefix() output '%v'was not a prefix of badgerKey() '%v'", string(needle), string(prefix)))
}
if len(prefix)+20 != len(needle) {
panic(fmt.Sprintf("badgerPrefix() output '%v'was 20 characters shorter than badgerKey() '%v'", string(needle), string(prefix)))
}
// validate assumption that badgerKeyExtractContainerKey() makes about strconv.ParseUint() error reporting;
// for distinguishing prefixes from full keys. Even if the shard number is so large that the prefix
// starts with a legitimate decimal number.
shouldNotParse := "12345123451234';key@"
containerKey, err := strconv.ParseUint(shouldNotParse, 10, 64)
if err == nil {
panic(fmt.Sprintf("strconv.ParseUint should have returned an error parsing this string '%v'; instead we got '%v'", shouldNotParse, containerKey))
}
// verify panic on submitting a prefix
func() {
defer func() {
r := recover()
if r == nil {
panic(fmt.Sprintf("should have seen panic on call to badgerKeyExtractContainerKey(prefix='%v')", prefix))
}
}()
badgerKeyExtractContainerKey(prefix) // should panic.
}()
}
func TestBadger_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit")
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
defer tx.Rollback()
putme := uint64(1<<16) + 3 // in the key:1 container
@ -461,7 +511,7 @@ func TestBadger_ContainerIterator_empty_iteration_loop(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
defer tx.Rollback()
putme := uint64(1<<16) + 3 // in the key:1 container
@ -511,7 +561,7 @@ func TestBadger_ForEach_on_one_bit(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
defer tx.Rollback()
bitvalue := uint64(42)
@ -570,7 +620,7 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// delete, but rollback instead of commit
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
hi := highbits(putme)
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
tx.Rollback()
@ -579,7 +629,7 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// c) within one Tx, after delete it should be gone as viewed within the txn.
tx = dbwrap.NewBadgerTx(writable, index)
tx = dbwrap.NewBadgerTx(writable, index, nil)
hi = highbits(putme)
exists, err := tx.Contains(index, field, view, shard, putme)
@ -631,7 +681,7 @@ func TestBadger_Remove_one_bit_test(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// delete, but rollback instead of commit
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
hi, lo := highbits(putme), lowbits(putme)
_, _ = hi, lo
_, err := tx.Remove(index, field, view, shard, hi)
@ -642,7 +692,7 @@ func TestBadger_Remove_one_bit_test(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// c) within one Tx, after delete it should be gone as viewed within the txn.
tx = dbwrap.NewBadgerTx(writable, index)
tx = dbwrap.NewBadgerTx(writable, index, nil)
exists, err := tx.Contains(index, field, view, shard, putme)
panicOn(err)
@ -717,31 +767,6 @@ func TestBadger_reverse_badger_iterator(t *testing.T) {
panicOn(err)
}
func TestBadger_Max_on_many_containers(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Max_on_many_containers")
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
putmeValues := []uint64{0, 2 << 16, 4 << 16}
for _, putme := range putmeValues {
badgerDBMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme)
badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
max, err := tx.Max(index, field, view, shard)
panicOn(err)
expected := putmeValues[len(putmeValues)-1]
if max != expected {
panic(fmt.Sprintf("expected Max() of %v but got max=%v", expected, max))
}
}
func TestBadger_Min_on_many_containers(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_Min_on_many_containers")
defer clean()
@ -749,7 +774,7 @@ func TestBadger_Min_on_many_containers(t *testing.T) {
index, field, view, shard := "i", "f", "v", uint64(0)
// verify no containers flag works
tx := dbwrap.NewBadgerTx(!writable, index)
tx := dbwrap.NewBadgerTx(!writable, index, nil)
min, containersExist, err := tx.Min(index, field, view, shard)
_ = min
panicOn(err)
@ -766,7 +791,7 @@ func TestBadger_Min_on_many_containers(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx = dbwrap.NewBadgerTx(!writable, index)
tx = dbwrap.NewBadgerTx(!writable, index, nil)
defer tx.Rollback()
min, containersExist, err = tx.Min(index, field, view, shard)
@ -787,7 +812,7 @@ func TestBadger_CountRange_on_many_containers(t *testing.T) {
index, field, view, shard := "i", "f", "v", uint64(0)
// verify no containers flag works
tx := dbwrap.NewBadgerTx(!writable, index)
tx := dbwrap.NewBadgerTx(!writable, index, nil)
n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
panicOn(err)
if n != 0 {
@ -803,7 +828,7 @@ func TestBadger_CountRange_on_many_containers(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx = dbwrap.NewBadgerTx(!writable, index)
tx = dbwrap.NewBadgerTx(!writable, index, nil)
defer tx.Rollback()
n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
@ -831,7 +856,7 @@ func TestBadger_CountRange_middle_container(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable, index)
tx := dbwrap.NewBadgerTx(!writable, index, nil)
defer tx.Rollback()
// pick out just the middle container with the 1 bit set on it.
@ -856,7 +881,7 @@ func TestBadger_CountRange_many_middle_container(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable, index)
tx := dbwrap.NewBadgerTx(!writable, index, nil)
defer tx.Rollback()
// get them all
@ -887,7 +912,7 @@ func TestBadger_UnionInPlace(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx2 := dbwrap.NewBadgerTx(!writable, index)
tx2 := dbwrap.NewBadgerTx(!writable, index, nil)
n, err := tx2.Count(index, field, view, shard)
panicOn(err)
if n != 2 {
@ -902,7 +927,7 @@ func TestBadger_UnionInPlace(t *testing.T) {
}
mustAddR(others3.Add(4 << 16)) // outside the 2<<16 container
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
defer tx.Rollback()
err = tx.UnionInPlace(index, field, view, shard, others, others2, others3)
panicOn(err)
@ -927,7 +952,7 @@ func TestBadger_RoaringBitmap(t *testing.T) {
putme := expected
badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
tx := dbwrap.NewBadgerTx(!writable, index)
tx := dbwrap.NewBadgerTx(!writable, index, nil)
defer tx.Rollback()
rbm, err := tx.RoaringBitmap(index, field, view, shard)
@ -959,7 +984,7 @@ func TestBadger_reverse_badger_iterator_and_prefix_valid(t *testing.T) {
})
panicOn(err)
tx := dbwrap.NewBadgerTx(!writable, "no-index-avail")
tx := dbwrap.NewBadgerTx(!writable, "no-index-avail", nil)
prefix := []byte("b:")
it := NewBadgerIterator(tx, prefix)
@ -1020,7 +1045,7 @@ func TestBadger_just_reverse_badger_iterator_and_prefix_valid(t *testing.T) {
})
panicOn(err)
tx := dbwrap.NewBadgerTx(!writable, "no-index-avail")
tx := dbwrap.NewBadgerTx(!writable, "no-index-avail", nil)
seekto := []byte("c:")
prefix := []byte("b:")
@ -1050,7 +1075,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
defer tx.Rollback()
tx.DeleteEmptyContainer = true // traditional badger Tx behavior, but not Roaring.
@ -1133,7 +1158,7 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
defer tx.Rollback()
// get some roaring bits, get an itr RoaringIterator from them
@ -1183,7 +1208,7 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
defer tx.Rollback()
// get some roaring bits, get an itr RoaringIterator from them
@ -1257,7 +1282,7 @@ func TestBadger_DeleteIndex(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
bitvalue := uint64(777)
bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16}
for _, v := range bits {
@ -1294,7 +1319,7 @@ func TestBadger_DeleteIndex(t *testing.T) {
err = dbwrap.DeleteIndex(index)
panicOn(err)
tx = dbwrap.NewBadgerTx(!writable, index2)
tx = dbwrap.NewBadgerTx(!writable, index2, nil)
defer tx.Rollback()
exists, err = tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
@ -1319,7 +1344,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) {
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
tx := dbwrap.NewBadgerTx(writable, index, nil)
bitvalue := uint64(777)
limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction.
//limit := uint64(101)
@ -1332,7 +1357,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) {
panicOn(err)
if v%100000 == 0 {
panicOn(tx.Commit())
tx = dbwrap.NewBadgerTx(writable, index)
tx = dbwrap.NewBadgerTx(writable, index, nil)
}
}
@ -1349,7 +1374,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) {
err = dbwrap.DeleteIndex(index)
panicOn(err)
tx = dbwrap.NewBadgerTx(!writable, index2)
tx = dbwrap.NewBadgerTx(!writable, index2, nil)
defer tx.Rollback()
exists, err := tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
@ -1450,92 +1475,6 @@ func mustRemove(changeCount int, err error) {
panicOn(err)
}
func TestBadger_DeleteFragment(t *testing.T) {
// setup
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteFragment")
defer clean()
defer dbwrap.Close()
index, field, view, shard0 := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
shard1 := uint64(1)
bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16}
shards := []uint64{shard0, shard1}
for _, s := range shards {
for _, v := range bits {
changed, err := tx.Add(index, field, view, s, doBatched, v)
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
}
}
for _, s := range shards {
for _, v := range bits {
exists, err := tx.Contains(index, field, view, s, v)
panicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!!")
}
}
}
err := tx.Commit()
panicOn(err)
// end of setup
survivor := shard0
victim := shard1
err = dbwrap.DeleteFragment(index, field, view, victim, nil)
panicOn(err)
tx = dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
for _, s := range shards {
for _, v := range bits {
exists, err := tx.Contains(index, field, view, s, v)
panicOn(err)
if s == survivor {
if !exists {
panic(fmt.Sprintf("ARG survivor died : bit %v", v))
}
} else if s == victim { // victim, should have been deleted
if exists {
panic(fmt.Sprintf("ARG victim lived : bit %v", v))
}
}
}
}
}
func TestBadger_shardFromBadgerKey(t *testing.T) {
if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615")) != 1 {
panic("problem")
}
if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'0';ckey@18446744073709551615")) != 0 {
panic("problem")
}
if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@18446744073709551615")) != 18446744073709551615 {
panic("problem")
}
func() {
defer func() {
r := recover()
if r == nil {
panic("should have panic-ed")
}
}()
// called for the panic of a short ckey, only 19 bytes instead of 20
shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@1844674407370955161"))
}()
}
func TestBadger_SliceOfShards(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SliceOfShards")
@ -1547,7 +1486,7 @@ func TestBadger_SliceOfShards(t *testing.T) {
for _, shard := range shards {
badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable, index)
tx := dbwrap.NewBadgerTx(!writable, index, nil)
defer tx.Rollback()
slc, err := tx.SliceOfShards(index, field, view, "")
@ -1559,6 +1498,89 @@ func TestBadger_SliceOfShards(t *testing.T) {
}
}
// Benchmark performance of setValue for BSI ranges.
func BenchmarkBadger_Write(b *testing.B) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("BenchmarkBadger_Write")
//defer clean()
_ = clean
defer dbwrap.Close()
putmeValues := []uint64{3, 2 << 16}
index, field, view, shard := "i", "f", "v", uint64(0)
for _, putme := range putmeValues {
badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
}
/*
dbwrap, clean := mustOpenEmptyBadgerWrapper("BenchmarkBadger_Write")
defer clean()
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index, nil)
bitvalue := uint64(1 << 20)
changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
bitvalue2 := uint64(1<<20 + 1)
changed, err = tx.Add(index, field, view, shard, doBatched, bitvalue2)
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!!")
}
exists, err = tx.Contains(index, field, view, shard, bitvalue2)
panicOn(err)
if !exists {
panic("ARG bitvalue2 was NOT SET!!!")
}
err = tx.Commit()
panicOn(err)
offset := uint64(0 << 20)
start := uint64(0 << 16)
endx := bitvalue + 1<<16
tx2 := dbwrap.NewBadgerTx(!writable, index, nil)
rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
tx2.Rollback()
// should see our 1M value
s2 := bitmapAsString(rbm2)
expect2 := "c(1048576, 1048577)"
if s2 != expect2 {
panic(fmt.Sprintf("s2='%v', but expected '%v'", s2, expect2))
}
// now offset by 2M
offset = uint64(2 << 20)
tx3 := dbwrap.NewBadgerTx(!writable, index, nil)
rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
tx3.Rollback()
//expect to see 3M == 3145728
s3 := bitmapAsString(rbm3)
expect3 := "c(3145728, 3145729)"
if s3 != expect3 {
panic(fmt.Sprintf("s3='%v', but expected '%v'", s3, expect3))
}
*/
}
func reportTestBadgersNeedingClose() {
globalBadgerReg.mu.Lock()
defer globalBadgerReg.mu.Unlock()

8
blake3.go
View file

@ -21,6 +21,7 @@ import (
cryptorand "crypto/rand"
"github.com/zeebo/blake3"
"golang.org/x/mod/sumdb/dirhash"
)
// Blake3Hasher is a thread/goroutine safe way to
@ -93,3 +94,10 @@ func cryptoRandInt64() int64 {
r := int64(binary.LittleEndian.Uint64(b))
return r
}
func HashOfDir(path string) string {
prefix := ""
h, err := dirhash.HashDir(path, prefix, dirhash.Hash1)
panicOn(err)
return h
}

24
blake3_test.go
View file

@ -16,6 +16,8 @@ package pilosa
import (
"fmt"
"io/ioutil"
"os"
"testing"
"encoding/hex"
@ -45,3 +47,25 @@ func TestCryptoRandInt64(t *testing.T) {
panic("cryptoRandInt64() gave 0, very high odds it has broken")
}
}
func TestHashOfDir(t *testing.T) {
dir, err := ioutil.TempDir(".", "TestHashOfDir-dir")
panicOn(err)
b := dir + sep + "A" + sep + "B"
c := dir + sep + "A" + sep + "C"
panicOn(os.MkdirAll(b, 0755))
panicOn(os.MkdirAll(c, 0755))
bmessage := []byte("hello B\n")
panicOn(ioutil.WriteFile(b+sep+"b_content", bmessage, 0644))
cmessage := []byte("hello C\n")
panicOn(ioutil.WriteFile(c+sep+"c_content", cmessage, 0644))
defer os.RemoveAll(dir)
hsh := HashOfDir(dir)
c2message := []byte("hello C2\n")
panicOn(ioutil.WriteFile(c+sep+"c_content", c2message, 0644))
hsh2 := HashOfDir(dir)
if hsh2 == hsh {
panic("HashOfDir did not detect 1 byte change")
}
}

28
bluegreentx.go
View file

@ -85,7 +85,8 @@ func (c *blueGreenTx) Readonly() bool {
func (c *blueGreenTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
c.checker.see(index, field, view, shard)
// TODO(jea): does this need to be different, to handle c.a iteration at the same time?
// can't really do simultaneous iteration on A and B, so punt and
// just give back B.
return c.b.NewTxIterator(index, field, view, shard)
}
@ -141,17 +142,20 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
if bKey != aKey {
AlwaysPrintf("problem in caller %v", Caller(2))
c.Dump()
panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, stack())) // crashing here on TestBSIGroup_importValue
panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, stack()))
}
if err := aValue.BitwiseCompare(bValue); err != nil {
c.Dump()
vv("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack())
panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack()))
}
}
// end checking everything in A, but does B have more?
if bIter.Next() {
bKey, _ := bIter.Value()
AlwaysPrintf("bIter has more than it should. problem in caller %v", Caller(2))
c.Dump()
bKey, _ := bIter.Value()
vv("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, at %v", here, c.bs, bKey, c.as, stack())
panic(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, at %v", here, c.bs, bKey, c.as, stack()))
}
}
@ -177,7 +181,7 @@ func (c *blueGreenTx) Rollback() {
c.mu.Lock()
defer c.mu.Unlock()
if c.rollbackOrCommitDone {
return
return // avoid using discarded tx for Dump, which will panic.
}
c.rollbackOrCommitDone = true
@ -188,13 +192,16 @@ func (c *blueGreenTx) Rollback() {
panic(r)
}
}()
fmt.Printf("blueGreenTx.Rollback() about to call (%v) a.Rollback()\n", c.as)
c.a.Rollback()
fmt.Printf("blueGreenTx.Rollback() about to call (%v) b.Rollback()\n", c.bs)
c.b.Rollback()
}
func (c *blueGreenTx) Commit() error {
c.mu.Lock()
defer c.mu.Unlock()
fmt.Printf("blueGreenTx.Commit() called.\n")
if c.rollbackOrCommitDone {
return nil
}
@ -332,7 +339,9 @@ func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, k
}
func (c *blueGreenTx) UseRowCache() bool {
return c.b.UseRowCache()
// avoid cross-talk between our two implementations
// by never allowing either to use the row cache.
return false
}
func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
@ -612,7 +621,10 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star
b, errB := c.b.OffsetRange(index, field, view, shard, offset, start, end)
err = roaringBitmapDiff(a, b)
panicOn(err)
if err != nil {
c.Dump()
panicOn(err)
}
compareErrors(errA, errB)
return b, errB
}
@ -731,7 +743,7 @@ func (m *MultiReaderB) Read(p []byte) (nB int, errB error) {
}
cmp := bytes.Compare(p[:nB], p2[:nB])
if cmp != 0 {
panic(fmt.Sprintf("MultiReaderB reads p and p2 (cmp= %v) differed.", cmp)) // \np ='%v'; \np2 ='%v'", cmp, string(p[:nB]), string(p2[:nA])))
panic(fmt.Sprintf("MultiReaderB reads p and p2 (cmp= %v) differed.", cmp))
}
}
return
@ -754,7 +766,7 @@ type blueGreenChecker struct {
// see would mark a thing as seen.
func (b *blueGreenChecker) see(index, field, view string, shard uint64) {
// keep this next Printf. Useful to see the sequence of Tx operations.
//fmt.Printf("blueGreenTx.%v\n", Caller(1))
fmt.Printf("blueGreenTx.%v on index='%v'\n", Caller(1), index)
b.mu.Lock()
defer b.mu.Unlock()

4
catcher.go
View file

@ -48,10 +48,6 @@ func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roar
return c.b.NewTxIterator(index, field, view, shard)
}
func (c *catcherTx) WholeDatabaseBlake3Hash(index, field, view string, shard uint64) (hash string, err error) {
return c.b.WholeDatabaseBlake3Hash(index, field, view, shard)
}
func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
defer func() {
if r := recover(); r != nil {

267
cmd/demo-lmdb/lmdb.go Normal file
View file

@ -0,0 +1,267 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !386
package main
import (
"bytes"
"fmt"
"log"
"os"
"github.com/glycerine/lmdb-go/lmdb"
)
// note: use runtime.LockOSThread on any write goroutine; must create and write the txn from
// the same goroutine.
// This example demonstrates a complete workflow for a simple application
// working with LMDB. First, an Env is configured and mapped to memory. Once
// mapped, database handles are opened and normal database operations may
// begin.
func main() {
// Create an environment and make sure it is eventually closed.
env, err := lmdb.NewEnv()
panicOn(err)
defer env.Close()
// Configure and open the environment. Most configuration must be done
// before opening the environment. The go documentation for each method
// should indicate if it must be called before calling env.Open()
err = env.SetMaxDBs(1)
panicOn(err)
err = env.SetMapSize(1 << 30)
panicOn(err)
path := "./db-lmdb"
panicOn(os.MkdirAll(path, 0755))
err = env.Open(path, 0, 0644) // lmdb.Create ?
panicOn(err)
// In any real application it is important to check for readers that were
// never closed by their owning process, and for which the owning process
// has exited. See the documentation on transactions for more information.
staleReaders, err := env.ReaderCheck()
panicOn(err)
if staleReaders > 0 {
log.Printf("cleared %d reader slots from dead processes", staleReaders)
}
// Open a database handle that will be used for the entire lifetime of this
// application. Because the database may not have existed before, and the
// database may need to be created, we need to get the database handle in
// an update transacation.
var dbi lmdb.DBI
_ = dbi
err = env.Update(func(txn *lmdb.Txn) (err error) {
dbi, err = txn.CreateDBI("example")
return err
})
panicOn(err)
// The database referenced by our DBI handle is now ready for the
// application to use. Here the application just opens a readonly
// transaction and reads the data stored in the "hello" key and prints its
// value to the application's standard output.
err = env.View(func(txn *lmdb.Txn) (err error) {
v, err := txn.Get(dbi, []byte("hello"))
if err != nil {
return err
}
fmt.Println(string(v))
return nil
})
_ = err
//panicOn(err) // mdb_get: MDB_NOTFOUND: No matching key/data pair found
err = env.Update(func(txn *lmdb.Txn) (err error) {
panicOn(txn.Put(dbi, []byte("099"), []byte("A"), 0))
panicOn(txn.Put(dbi, []byte("101"), []byte("B"), 0))
panicOn(txn.Put(dbi, []byte("199"), []byte("C"), 0))
panicOn(txn.Put(dbi, []byte("200"), []byte("D"), 0))
panicOn(txn.Put(dbi, []byte("300"), []byte("E"), 0))
panicOn(txn.Put(dbi, []byte("399"), []byte("F"), 0))
return nil
})
_ = err
// find max in [000,100) and get 099
// find max in [100,200) and get 199
// find max in [300,400) and get 399
// find max in [400,500) and get nothing back
err = env.View(func(txn *lmdb.Txn) (err error) {
v, err := txn.Get(dbi, []byte("hello"))
if err != nil {
return err
}
fmt.Printf("key 'hello' retreived value: '%v'\n", string(v))
return nil
})
_ = err
err = env.View(func(txn *lmdb.Txn) (err error) {
cur, err := txn.OpenCursor(dbi)
panicOn(err)
defer cur.Close()
var cur2 *lmdb.Cursor
var err2 error
var k, k2, v, v2 []byte
i := 0
for {
if i == 0 {
// lmdb.SetRange : The first key no less than the specified key.
k, v, err = cur.Get([]byte("200"), nil, lmdb.SetRange)
// cur2 should start at 'a'
cur2, err2 = txn.OpenCursor(dbi)
panicOn(err2)
defer cur2.Close()
k2, v2, err2 = cur2.Get(nil, nil, lmdb.Next)
_ = err2
} else {
k, v, err = cur.Get(nil, nil, lmdb.Next)
k2, v2, err2 = cur2.Get(nil, nil, lmdb.Next)
_ = err2
}
if lmdb.IsNotFound(err) {
return nil
}
if err != nil {
return err
}
fmt.Printf("i=%v, %s %s\n", i, k, v)
fmt.Printf("i=%v, k2:%s v2:%s\n", i, k2, v2)
i++
}
// return nil // unreachable
})
_ = err
// panicOn(txn.Put(dbi, []byte("099"), []byte("A"), 0))
// panicOn(txn.Put(dbi, []byte("101"), []byte("B"), 0))
// panicOn(txn.Put(dbi, []byte("199"), []byte("C"), 0))
// panicOn(txn.Put(dbi, []byte("200"), []byte("D"), 0))
// panicOn(txn.Put(dbi, []byte("300"), []byte("E"), 0))
// panicOn(txn.Put(dbi, []byte("399"), []byte("F"), 0))
//
// find max in [300,400) and get 399
// find max in [000,100) and get 099
// find max in [100,200) and get 199
// find max in [400,500) and get nothing back
// find max in [201,300) and get nothing back
err = env.View(func(txn *lmdb.Txn) (err error) {
cur, err := txn.OpenCursor(dbi)
panicOn(err)
defer cur.Close()
var k, v []byte
// find max in [300,400) and get 399
// lmdb.SetRange : The first key no less than the specified key.
k, v, err = cur.Get([]byte("400"), nil, lmdb.SetRange)
if lmdb.IsNotFound(err) {
fmt.Printf("400 not found, as expected\n") // happens on starting empty db
} else {
fmt.Printf("Get 400 => %v: %v\n", string(k), string(v))
}
k, v, err = cur.Get(nil, nil, lmdb.Prev)
if lmdb.IsNotFound(err) {
fmt.Printf("Get 400 then Get Prev => not found\n")
} else {
fmt.Printf("Get 400 then Get Prev => %v: %v\n", string(k), string(v)) // 399: F, so wraps backwards from beginning.
}
panicOn(err)
// now try for 199 in [100,200)
k, v, err = cur.Get([]byte("200"), nil, lmdb.SetRange)
if lmdb.IsNotFound(err) {
panic("200 not found, not expected")
} else {
fmt.Printf("Get 200 => %v: %v\n", string(k), string(v)) // Get 200 => 200: D
}
k, v, err = cur.Get(nil, nil, lmdb.Prev)
if lmdb.IsNotFound(err) {
fmt.Printf("Get 200 then Get Prev => not found\n")
} else {
fmt.Printf("Get 200 then Get Prev => %v: %v\n", string(k), string(v)) // Get 200 then Get Prev => 199: C
}
panicOn(err)
k, v, err = cur.Get([]byte("500"), nil, lmdb.SetRange)
if lmdb.IsNotFound(err) {
fmt.Printf("500 not found, as expected\n") // 500 not found, as expected
} else {
panic(fmt.Sprintf("Get 500 => %v: %v\n", string(k), string(v)))
}
k, v, err = cur.Get(nil, nil, lmdb.Prev)
if lmdb.IsNotFound(err) {
fmt.Printf("Get 500 then Get Prev => not found\n")
} else {
fmt.Printf("Get 500 then Get Prev => %v: %v\n", string(k), string(v)) // Get 500 then Get Prev => 399: F
}
panicOn(err)
k, v, err = cur.Get([]byte("100"), nil, lmdb.SetRange)
if lmdb.IsNotFound(err) {
panic("100 not found, not expected")
} else {
fmt.Printf("Get 100 => %v: %v\n", string(k), string(v)) // Get 100 => 101: B
}
k, v, err = cur.Get(nil, nil, lmdb.Prev)
if lmdb.IsNotFound(err) {
fmt.Printf("Get 100 then Get Prev => not found\n")
} else {
fmt.Printf("Get 100 then Get Prev => %v: %v\n", string(k), string(v)) // Get 100 then Get Prev => 099: A
}
panicOn(err)
// find max in [201,300) and get nothing back
k, v, err = cur.Get([]byte("300"), nil, lmdb.SetRange)
if lmdb.IsNotFound(err) {
panic("300 not found, not expected")
} else {
fmt.Printf("Get 300 => %v: %v\n", string(k), string(v)) // Get 300 => 300: E
}
k, v, err = cur.Get(nil, nil, lmdb.Prev)
if lmdb.IsNotFound(err) {
fmt.Printf("Get 300 then Get Prev => not found\n")
} else {
fmt.Printf("Get 300 then Get Prev => %v: %v\n", string(k), string(v)) // Get 300 then Get Prev => 200: D
}
cmp := bytes.Compare(k, []byte("201"))
if cmp >= 0 {
fmt.Printf("key k = '%v' was >= 201", string(k))
} else {
fmt.Printf("key k = '%v' was < 201", string(k)) // key k = '200' was < 201
}
panicOn(err)
return nil
})
panicOn(err)
vv("done")
}

179
cmd/demo-lmdb/vprint.go Normal file
View file

@ -0,0 +1,179 @@
// home: https://github.com/glyerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
// +build !386
package main
import (
"fmt"
"io"
"os"
"path"
"runtime"
"runtime/debug"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("\n%s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}
// happy linter:
var _ = DirExists
var _ = FileExists
var _ = Caller
var _ = stack
var _ = RFC3339MsecTz0
var _ = RFC3339UsecTz0
var _ = AlwaysPrintf
var _ = FileSize

150
cmd/loader/loader.go
View file

@ -1,150 +0,0 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"archive/tar"
"compress/gzip"
"context"
"time"
//"fmt"
"fmt"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/http"
"io"
"io/ioutil"
gohttp "net/http"
//"log"
"os"
//"path/filepath"
//"sort"
"strconv"
"strings"
)
func UploadTar(srcFile string, client *http.InternalClient) error {
t0 := time.Now()
f, err := os.Open(srcFile)
if err != nil {
return (err)
}
defer f.Close()
var tarReader *tar.Reader
if strings.HasSuffix(srcFile, "gz") {
gzf, err := gzip.NewReader(f)
if err != nil {
return err
}
tarReader = tar.NewReader(gzf)
} else {
tarReader = tar.NewReader(f)
}
viewData := make(map[string][]byte)
//given ordered by index/field/view
//trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255
lastIndex := ""
lastField := ""
lastShard := uint64(0)
//vv("top of tar loop")
n := 0
for {
header, err := tarReader.Next()
if err == io.EOF {
if header != nil {
panic("header should not be nil on err io.EOF")
}
//submit any stuff we have left
if len(viewData) > 0 {
request := &pilosa.ImportRoaringRequest{
Views: viewData,
}
// Submit(lastIndex, lastField, lastShard, request)
//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
uri := GetImportRoaringURI(lastIndex, lastShard)
err := client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)
panicOn(err)
//vv("done with submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
}
return nil
}
//vv("got header '%v'", header.Name)
n++
if n%500 == 0 {
vv("n = %v, progress, elapsed '%v'", n, time.Since(t0))
}
parts := strings.Split(header.Name, "/")
index := parts[0]
field := parts[1]
view := parts[3]
shard, err := strconv.ParseUint(parts[5], 10, 64)
if err != nil {
return err
}
// TODO: shards can be loaded in parallel, so maybe farm out to a worker set of goro.
if index != lastIndex || field != lastField || shard != lastShard {
if len(viewData) > 0 {
request := &pilosa.ImportRoaringRequest{
Views: viewData,
}
//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
uri := GetImportRoaringURI(lastIndex, lastShard)
panicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request))
viewData = make(map[string][]byte)
//vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0))
}
}
roaringData, err := ioutil.ReadAll(tarReader)
if err != nil {
return err
}
if _, already := viewData[view]; already {
panic(fmt.Sprintf("view '%v' already present!", view))
}
viewData[view] = roaringData
lastIndex = index
lastField = field
//lastShard = shard
//vv("bottom of loop")
}
}
func main() {
host := "127.0.0.1:10101"
h := &gohttp.Client{}
c, err := http.NewInternalClient(host, h)
panicOn(err)
tarSrcPath := "q2.tar.gz"
t0 := time.Now()
panicOn(UploadTar(tarSrcPath, c))
vv("total elapsed '%v'", time.Since(t0))
}
var globURI *pilosa.URI
func init() {
var err error
globURI, err = pilosa.NewURIFromHostPort("127.0.0.1", 10101)
panicOn(err)
}
// get correct node to go to.
func GetImportRoaringURI(index string, shard uint64) *pilosa.URI {
return globURI
}

205
cmd/slurp/slurp.go Normal file
View file

@ -0,0 +1,205 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"time"
//"fmt"
"fmt"
"io"
"io/ioutil"
gohttp "net/http"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/http"
//"log"
"os"
//"path/filepath"
//"sort"
"strconv"
"strings"
)
// slurp: slurp is a load-tester for importing bulk data.
// It allows us to measure write performance.
type stateMachine struct {
viewData map[string][]byte
lastIndex string
lastField string
lastShard uint64
state string
client *http.InternalClient
start time.Time
}
func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
parts := strings.Split(h.Name, "/")
switch parts[0] {
case "roaring":
index := parts[1]
field := parts[2]
view := parts[4]
shard, err := strconv.ParseUint(parts[6], 10, 64)
panicOn(err)
if index != r.lastIndex || field != r.lastField || shard != r.lastShard {
err := r.Upload()
if err != nil {
return err
}
}
roaringData, err := ioutil.ReadAll(tr)
if err != nil {
return err
}
if _, already := r.viewData[view]; already {
panic(fmt.Sprintf("view '%v' already present!", view))
}
r.viewData[view] = roaringData
r.lastIndex = index
r.lastField = field
r.lastShard = shard
case "bolt":
if r.state == "roaring" {
err := r.Upload()
if err != nil {
return err
}
vv("Finished import %v", time.Since(r.start))
}
//
uri := GetImportRoaringURI(r.lastIndex, r.lastShard)
switch v := parts[len(parts)-1]; v {
case "keys":
index := parts[1]
fieldName := parts[2]
if fieldName == "_keys" {
//skip index keys are not not real fields so will have no need for field keys
return nil
}
byteData, err := ioutil.ReadAll(tr)
panicOn(err)
br := bytes.NewReader(byteData)
err = r.client.ImportFieldKeys(context.Background(), uri, index, fieldName, false, br)
if err != nil {
return err
}
default:
pilosa.VV("%v", h.Name)
index := parts[1]
partition, err := strconv.ParseUint(v, 10, 64)
if err != nil {
return err
}
byteData, err := ioutil.ReadAll(tr)
panicOn(err)
br := bytes.NewReader(byteData)
err = r.client.ImportIndexKeys(context.Background(), uri, index, int(partition), false, br)
if err != nil {
return err
}
}
}
r.state = parts[0]
return nil
}
func (r *stateMachine) Upload() error {
if len(r.viewData) > 0 {
request := &pilosa.ImportRoaringRequest{
Views: r.viewData,
}
uri := GetImportRoaringURI(r.lastIndex, r.lastShard)
err := r.client.ImportRoaring(context.Background(), uri, r.lastIndex, r.lastField, r.lastShard, false, request)
if err != nil {
return err
}
r.viewData = make(map[string][]byte)
}
return nil
}
func UploadTar(srcFile string, client *http.InternalClient) error {
f, err := os.Open(srcFile)
if err != nil {
return (err)
}
defer f.Close()
var tarReader *tar.Reader
if strings.HasSuffix(srcFile, "gz") {
gzf, err := gzip.NewReader(f)
if err != nil {
return err
}
tarReader = tar.NewReader(gzf)
} else {
tarReader = tar.NewReader(f)
}
runner := &stateMachine{
viewData: make(map[string][]byte),
start: time.Now(),
}
runner.client = client
for {
header, err := tarReader.Next()
if err == io.EOF {
_ = runner.Upload()
break
}
if err != nil {
panicOn(err)
}
err = runner.NewHeader(header, tarReader)
panicOn(err)
}
return nil
}
func main() {
host := "127.0.0.1:10101"
h := &gohttp.Client{}
c, err := http.NewInternalClient(host, h)
panicOn(err)
tarSrcPath := os.Args[1] //"q2.tar.gz"
t0 := time.Now()
println("uploading", tarSrcPath)
panicOn(UploadTar(tarSrcPath, c))
vv("total elapsed '%v'", time.Since(t0))
}
var globURI *pilosa.URI
func init() {
var err error
globURI, err = pilosa.NewURIFromHostPort("127.0.0.1", 10101)
panicOn(err)
}
// get correct node to go to.
func GetImportRoaringURI(index string, shard uint64) *pilosa.URI {
return globURI
}

0
cmd/loader/vprint.go → cmd/slurp/vprint.go
View file

14
executor.go
View file

@ -243,9 +243,11 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
// Must copy out of Tx data before Commiting, because it will become invalid afterwards.
respSafeNoTxData := e.safeCopy(resp)
// Commit transaction.
if err := tx.Commit(); err != nil {
return respSafeNoTxData, err
// Commit transaction if writing; else let the defer Rollback have it.
if needWriteTxn {
if err := tx.Commit(); err != nil {
return respSafeNoTxData, err
}
}
return respSafeNoTxData, nil
}
@ -503,7 +505,8 @@ func (e *executor) execute(ctx context.Context, tx Tx, index string, q *pql.Quer
// still need to handle them. Since everything else was
// already precomputed by handlePreCallChildren, though,
// we don't need this logic in executeCall.
if newIndex := call.CallIndex(); newIndex != "" && newIndex != index {
newIndex := call.CallIndex()
if newIndex != "" && newIndex != index {
v, err = e.executeCall(ctx, tx, newIndex, call, nil, opt)
} else {
v, err = e.executeCall(ctx, tx, index, call, shards, opt)
@ -815,7 +818,7 @@ func (e *executor) executeCall(ctx context.Context, tx Tx, index string, c *pql.
return e.executeFieldValueCall(ctx, tx, index, c, shards, opt)
case "Precomputed":
return e.executePrecomputedCall(ctx, tx, index, c, shards, opt)
default:
default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap.
statFn()
return e.executeBitmapCall(ctx, tx, index, c, shards, opt)
}
@ -1374,6 +1377,7 @@ func (e *executor) executePrecomputedCall(ctx context.Context, tx Tx, index stri
// executeBitmapCall executes a call that returns a bitmap.
func (e *executor) executeBitmapCall(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCall")
span.LogKV("pqlCallName", c.Name)
defer span.Finish()

1
executor_test.go
View file

@ -5209,7 +5209,6 @@ func TestExecutor_ForeignIndex(t *testing.T) {
}
join := c.Query(t, "parent", `Intersect(Row(general=3), Distinct(Row(color="blue"), index="child", field="parent_id"))`).Results[0].(*pilosa.Row)
if !reflect.DeepEqual(join.Keys, []string{"one"}) {
t.Fatalf("unexpected keys: %v", join.Keys)
}

21
extensions/distinct.go
View file

@ -1,21 +0,0 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build plugindistinct
package extensions
import (
_ "github.com/molecula/extensions/distinct"
)

25
field.go
View file

@ -716,6 +716,7 @@ var fieldQueue = make(chan struct{}, 16)
// openViews opens and initializes the views inside the field.
func (f *Field) openViews() error {
file, err := os.Open(filepath.Join(f.path, "views"))
if os.IsNotExist(err) {
return nil
@ -754,17 +755,19 @@ fileLoop:
return fmt.Errorf("opening view: view=%s, err=%s", view.name, err)
}
// Automatically upgrade BSI v1 fragments if they exist & reopen view.
if bsig := f.bsiGroup(f.name); bsig != nil {
if ok, err := upgradeViewBSIv2(view, bsig.BitDepth); err != nil {
return errors.Wrap(err, "upgrade view bsi v2")
} else if ok {
if err := view.close(); err != nil {
return errors.Wrap(err, "closing upgraded view")
}
view = f.newView(f.viewPath(name), name)
if err := view.open(); err != nil {
return fmt.Errorf("re-opening view: view=%s, err=%s", view.name, err)
if f.idx.Txf.TxType() == roaringFragmentFilesTxn {
// Automatically upgrade BSI v1 fragments if they exist & reopen view.
if bsig := f.bsiGroup(f.name); bsig != nil {
if ok, err := upgradeViewBSIv2(view, bsig.BitDepth); err != nil {
return errors.Wrap(err, "upgrade view bsi v2")
} else if ok {
if err := view.close(); err != nil {
return errors.Wrap(err, "closing upgraded view")
}
view = f.newView(f.viewPath(name), name)
if err := view.open(); err != nil {
return fmt.Errorf("re-opening view: view=%s, err=%s", view.name, err)
}
}
}
}

55
field_internal_test.go
View file

@ -877,3 +877,58 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
})
}
}
func TestBSIGroup_TxReopenDB(t *testing.T) {
f := OpenField(t, OptFieldTypeInt(-100, 200))
defer f.Close()
options := &ImportOptions{}
for i, tt := range []struct {
columnIDs []uint64
values []int64
checkVal int64
expCols []uint64
}{
{
[]uint64{100},
[]int64{1},
1,
[]uint64{100},
},
{
[]uint64{100},
[]int64{8},
8,
[]uint64{100},
},
{
[]uint64{100},
[]int64{1},
1,
[]uint64{100},
},
} {
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
// can't do this, we are in a loop, not a function:
// defer tx.Rollback()
if err := f.importValue(tx, tt.columnIDs, tt.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(tx.Commit())
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
// no, same reason as above: defer tx.Rollback()
if row, err := f.Range(tx, f.name, pql.EQ, tt.checkVal); err != nil {
t.Fatalf("test %d, getting range: %s", i, err.Error())
} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
}
tx.Rollback()
} // loop
// the test: can we re-open a BSI fragment under badger/rbf.
_ = f.Reopen()
}

1
fragment.go
View file

@ -574,6 +574,7 @@ func (f *fragment) mustRow(tx Tx, rowID uint64) *Row {
// unprotectedRow returns a row from the row cache if available or from storage
// (updating the cache).
func (f *fragment) unprotectedRow(tx Tx, rowID uint64) (*Row, error) {
useRowCache := tx.UseRowCache()
if useRowCache {
r, ok := f.rowCache.Fetch(rowID)

11
fragment_internal_test.go
View file

@ -1786,7 +1786,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
// Ensure a fragment's cache can be persisted between restarts.
func TestFragment_RankCache_Persistence(t *testing.T) {
skipForRBF(t)
roaringOnlyTest(t)
index := mustOpenIndex(IndexOptions{})
defer index.Close()
@ -5329,6 +5329,9 @@ func TestImportValueConcurrent(t *testing.T) {
"blueGreenTx because the lack of transactional consistency " +
"from Roaring-per-file will create false comparison " +
"failures."))
case lmdbTxn:
t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " +
"lmdb since only a single writer is allowed at once."))
}
// Since eg.Go gets called multiple times below, each
@ -5646,9 +5649,3 @@ func TestFragment_Bug_Q2DoubleDelete(t *testing.T) {
t.Fatalf("expected nothing got %v", res)
}
}
func skipForRBF(tb testing.TB) {
if os.Getenv("PILOSA_TXSRC") == "rbf" {
tb.Skip("skip for RBF")
}
}

160
gid.go Normal file
View file

@ -0,0 +1,160 @@
// Copyright (c) 2014 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package pilosa
import (
"bytes"
"errors"
"fmt"
"runtime"
"strconv"
"sync"
)
// Sourced https://github.com/bradfitz/http2/blob/dc0c5c000ec33e263612939744d51a3b68b9cece/gotrack.go
var goroutineSpace = []byte("goroutine ")
var littleBuf = sync.Pool{
New: func() interface{} {
buf := make([]byte, 64)
return &buf
},
}
var _ = curGID // happy linter
func curGID() uint64 {
bp := littleBuf.Get().(*[]byte)
defer littleBuf.Put(bp)
b := *bp
b = b[:runtime.Stack(b, false)]
// Parse the 4707 out of "goroutine 4707 ["
b = bytes.TrimPrefix(b, goroutineSpace)
i := bytes.IndexByte(b, ' ')
if i < 0 {
panic(fmt.Sprintf("No space found in %q", b))
}
b = b[:i]
n, err := parseUintBytes(b, 10, 64)
if err != nil {
panic(fmt.Sprintf("Failed to parse goroutine ID out of %q: %v", b, err))
}
return n
}
// parseUintBytes is like strconv.ParseUint, but using a []byte.
func parseUintBytes(s []byte, base int, bitSize int) (n uint64, err error) {
var cutoff, maxVal uint64
if bitSize == 0 {
bitSize = int(strconv.IntSize)
}
s0 := s
switch {
case len(s) < 1:
err = strconv.ErrSyntax
return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
case 2 <= base && base <= 36:
// valid base; nothing to do
case base == 0:
// Look for octal, hex prefix.
switch {
case s[0] == '0' && len(s) > 1 && (s[1] == 'x' || s[1] == 'X'):
base = 16
s = s[2:]
if len(s) < 1 {
err = strconv.ErrSyntax
return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
}
case s[0] == '0':
base = 8
default:
base = 10
}
default:
err = errors.New("invalid base " + strconv.Itoa(base))
return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
}
n = 0
cutoff = cutoff64(base)
maxVal = 1<<uint(bitSize) - 1
for i := 0; i < len(s); i++ {
var v byte
d := s[i]
switch {
case '0' <= d && d <= '9':
v = d - '0'
case 'a' <= d && d <= 'z':
v = d - 'a' + 10
case 'A' <= d && d <= 'Z':
v = d - 'A' + 10
default:
n = 0
err = strconv.ErrSyntax
return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
}
if int(v) >= base {
n = 0
err = strconv.ErrSyntax
return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
}
if n >= cutoff {
// n*base overflows
n = 1<<64 - 1
err = strconv.ErrRange
return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
}
n *= uint64(base)
n1 := n + uint64(v)
if n1 < n || n1 > maxVal {
// n+v overflows
n = 1<<64 - 1
err = strconv.ErrRange
return n, &strconv.NumError{Func: "ParseUint", Num: string(s0), Err: err}
}
n = n1
}
return n, nil
}
// Return the first number n such that n*base >= 1<<64.
func cutoff64(base int) uint64 {
if base < 2 {
return 0
}
return (1<<64-1)/uint64(base) + 1
}

9
go.mod
View file

@ -12,15 +12,18 @@ require (
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
github.com/davecgh/go-spew v1.1.1
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311
github.com/glycerine/lmdb-go v1.9.11
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/gogo/protobuf v1.2.0
github.com/golang/protobuf v1.3.3
github.com/google/go-cmp v0.2.0
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
github.com/gorilla/handlers v1.3.0
github.com/gorilla/mux v1.7.0
github.com/hashicorp/memberlist v0.1.3
github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2 // indirect
github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/opentracing/opentracing-go v1.1.0
github.com/pelletier/go-toml v1.2.0
github.com/pkg/errors v0.8.1
@ -38,7 +41,7 @@ require (
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
github.com/zeebo/blake3 v0.0.4
go.uber.org/atomic v1.4.0 // indirect
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
golang.org/x/mod v0.3.0
golang.org/x/sync v0.0.0-20190423024810-112230192c58
golang.org/x/text v0.3.2 // indirect
google.golang.org/grpc v1.28.0

27
go.sum
View file

@ -51,6 +51,12 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8=
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E=
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y=
github.com/glycerine/lmdb-go v1.9.11 h1:Jutsg5jgYxZIHf5DqV4Bu+JVYs3Ieax7DASNigr6TUg=
github.com/glycerine/lmdb-go v1.9.11/go.mod h1:iztA3wBlR0RO8jTYTqGTGoySIEa6vFAXWEByWusDfOY=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
@ -77,6 +83,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCy
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI=
github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
@ -99,6 +107,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@ -116,14 +126,6 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
github.com/molecula/ext v0.0.0-20191202195653-240f38a75171 h1:4VK7u/RM+54Yaz8aRB9vIaDSnbKi3M0NQYg5tsZvOT4=
github.com/molecula/ext v0.0.0-20191202195653-240f38a75171/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ=
github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2 h1:XOImsA5XhGklFj8Y0TxSm1qWZzEwYxom2JOXiu9GMq0=
github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2/go.mod h1:r6EIj0GH8dx5xxFLW6Voi1/mX3wXOUkJu6AoEE/xvGQ=
github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4 h1:mDB/dicofRVFuRYcCVPk+JBiVKXlfbzMahuqHvrYqu4=
github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4/go.mod h1:QQgN5OFjuBAi4Q2UYVMzfvi4k9yvg/qqC+MNFB4I9JI=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
@ -210,12 +212,14 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM=
@ -256,6 +260,9 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=

10
holder.go
View file

@ -511,8 +511,10 @@ func (h *Holder) Open() error {
if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
continue
}
// Skip badgerdb files too.
if strings.HasSuffix(fi.Name(), "badgerdb") {
// Skip embedded db files too.
if strings.HasSuffix(fi.Name(), "-badgerdb") ||
strings.HasSuffix(fi.Name(), "-lmdb") ||
strings.HasSuffix(fi.Name(), "-rbfdb") {
continue
}
@ -611,6 +613,9 @@ func (h *Holder) Close() error {
if err := index.Close(); err != nil {
return errors.Wrap(err, "closing index")
}
if err := index.Txf.CloseDB(); err != nil {
return errors.Wrap(err, "index.Txf.CloseDB()")
}
}
// Reset opened in case Holder needs to be reopened.
@ -1498,7 +1503,6 @@ func (s *holderSyncer) initializeIndexTranslateReplication() error {
if !index.Keys() {
continue
}
for partitionID := 0; partitionID < s.Cluster.partitionN; partitionID++ {
partitionNodes := s.Cluster.partitionNodes(partitionID)
isPrimary := partitionNodes[0].ID == node.ID // remote is primary?

1
holder_test.go
View file

@ -403,6 +403,7 @@ func TestHolder_HasData(t *testing.T) {
// Ensure holder can delete an index and its underlying files.
func TestHolder_DeleteIndex(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()

63
http/client.go
View file

@ -1685,3 +1685,66 @@ func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context,
return resp.Body, nil
}
func (c *InternalClient) ImportIndexKeys(ctx context.Context, uri *pilosa.URI, index string, partitionID int, remote bool, rddbdata io.Reader) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportIndexKeys")
defer span.Finish()
if index == "" {
return pilosa.ErrIndexRequired
}
if uri == nil {
uri = c.defaultURI
}
vals := url.Values{}
vals.Set("remote", strconv.FormatBool(remote))
url := fmt.Sprintf("%s/internal/translate/index/%s/%d", uri, index, partitionID)
// Generate HTTP request.
httpReq, err := http.NewRequest("POST", url, rddbdata)
if err != nil {
return errors.Wrap(err, "creating request")
}
httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
// Execute request against the host.
resp, err := c.executeRequest(httpReq.WithContext(ctx))
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pilosa.URI, index, field string, remote bool, rddbdata io.Reader) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.ImportFieldKeys")
defer span.Finish()
if index == "" {
return pilosa.ErrIndexRequired
}
if uri == nil {
uri = c.defaultURI
}
vals := url.Values{}
vals.Set("remote", strconv.FormatBool(remote))
url := fmt.Sprintf("%s/internal/translate/field/%s/%s", uri, index, field)
// Generate HTTP request.
httpReq, err := http.NewRequest("POST", url, rddbdata)
if err != nil {
return errors.Wrap(err, "creating request")
}
httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
// Execute request against the host.
resp, err := c.executeRequest(httpReq.WithContext(ctx))
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}

106
http/handler.go
View file

@ -28,8 +28,10 @@ import (
"net/http"
_ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server.
"net/url"
"os"
"reflect"
"runtime/debug"
"runtime/pprof"
"strconv"
"strings"
"sync"
@ -44,6 +46,7 @@ import (
"github.com/pilosa/pilosa/v2/tracing"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/zeebo/blake3"
)
// Handler represents an HTTP handler.
@ -385,6 +388,9 @@ func newRouter(handler *Handler) *mux.Router {
router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes")
router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client
router.HandleFunc("/internal/translate/index/{index}/{partition}", handler.handlePostTranslateIndexDB).Methods("POST").Name("PostTranslateIndexDB")
router.HandleFunc("/internal/translate/field/{index}/{field}", handler.handlePostTranslateFieldDB).Methods("POST").Name("PostTranslateFieldDB")
router.Use(handler.queryArgValidator)
router.Use(handler.addQueryContext)
router.Use(handler.extractTracing)
@ -634,14 +640,55 @@ type getStatusResponse struct {
LocalID string `json:"localID"`
}
func hash(s string) string {
hasher := blake3.New()
_, _ = hasher.Write([]byte(s))
var buf [16]byte
_, _ = hasher.Digest().Read(buf[0:])
return fmt.Sprintf("%x", buf)
}
var DoPerQueryProfiling = false
// handlePostQuery handles /query requests.
func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
// Read previouly parsed request from context
qreq := r.Context().Value(contextKeyQueryRequest)
qerr := r.Context().Value(contextKeyQueryError)
req, ok := qreq.(*pilosa.QueryRequest)
err, _ := qerr.(error)
if DoPerQueryProfiling {
txsrc := os.Getenv("PILOSA_TXSRC")
reqHash := hash(req.Query)
qlen := len(req.Query)
if qlen > 100 {
qlen = 100
}
name := "_query." + reqHash + "." + txsrc + "." + time.Now().Format("20060102150405") + "." + req.Query[:qlen]
f, err := os.Create(name)
if err != nil {
panic(err)
}
defer f.Close()
_ = pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
} // end DoPerQueryProfiling
/*
er = trace.Start(f)
if er != nil {
panic(er)
}
defer trace.Stop()
*/
var err error
err, _ = qerr.(error)
if err != nil || !ok {
w.WriteHeader(http.StatusBadRequest)
@ -2203,3 +2250,58 @@ func readBody(r *http.Request) ([]byte, error) {
return buf.Bytes(), nil
}
func (h *Handler) handlePostTranslateFieldDB(w http.ResponseWriter, r *http.Request) {
indexName, ok := mux.Vars(r)["index"]
if !ok {
http.Error(w, "index name is required", http.StatusBadRequest)
return
}
fieldName, ok := mux.Vars(r)["field"]
if !ok {
http.Error(w, "field name is required", http.StatusBadRequest)
return
}
bd, err := readBody(r)
if err != nil {
http.Error(w, "failed to read body", http.StatusBadRequest)
return
}
br := bytes.NewReader(bd)
err = h.api.TranslateFieldDB(r.Context(), indexName, fieldName, br)
resp := successResponse{h: h, Name: fieldName}
resp.check(err)
resp.write(w, err)
}
func (h *Handler) handlePostTranslateIndexDB(w http.ResponseWriter, r *http.Request) {
indexName, ok := mux.Vars(r)["index"]
if !ok {
http.Error(w, "index name is required", http.StatusBadRequest)
return
}
partitionArg, ok := mux.Vars(r)["partition"]
if !ok {
http.Error(w, "partition is required", http.StatusBadRequest)
return
}
partition, err := strconv.ParseUint(partitionArg, 10, 64)
if err != nil {
http.Error(w, "bad partition", http.StatusBadRequest)
return
}
bd, err := readBody(r)
if err != nil {
http.Error(w, "failed to read body", http.StatusBadRequest)
return
}
br := bytes.NewReader(bd)
err = h.api.TranslateIndexDB(r.Context(), indexName, int(partition), br)
resp := successResponse{h: h, Name: indexName}
resp.check(err)
resp.write(w, err)
}

31
index.go
View file

@ -33,6 +33,16 @@ import (
"golang.org/x/sync/errgroup"
)
// debug: TODO(jea): remove this init() that does cpu profiling.
func init() {
go func() {
// give time for env var TXSRC to be set.
//time.Sleep(5 * time.Second)
//CPUProfileForDur(5*time.Minute, "cpu.pprof")
//CPUProfileForDur(15*time.Second, "cpu.pprof")
}()
}
// Index represents a container for fields.
type Index struct {
mu sync.RWMutex
@ -72,21 +82,9 @@ type Index struct {
Txf *TxFactory
}
// OpenIndex opens or starts a new Index on path. Path
// can be empty.
func OpenIndex(holder *Holder, path, name string) (*Index, error) {
openExisting := true
return openOrCreateNewIndex(holder, path, name, openExisting)
}
// NewIndex returns a new instance of Index at path. It will erase anything
// old already in path.
// NewIndex returns an existing (but possibly empty) instance of
// Index at path. It will not erase any prior content.
func NewIndex(holder *Holder, path, name string) (*Index, error) {
openExisting := false
return openOrCreateNewIndex(holder, path, name, openExisting)
}
func openOrCreateNewIndex(holder *Holder, path, name string, openExisting bool) (*Index, error) {
// Emulate what the spf13/cobra does, letting env vars override
// the defaults, because we may be under a simple "go test" run where
@ -117,7 +115,7 @@ func openOrCreateNewIndex(holder *Holder, path, name string, openExisting bool)
return nil, errors.Wrap(err, "validating name")
}
txf, err := NewTxFactory(txsrc, holder.Path, name, openExisting)
txf, err := NewTxFactory(txsrc, holder.Path, name)
if err != nil {
return nil, errors.Wrap(err, "creating newTxFactory")
}
@ -252,6 +250,9 @@ func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) {
mu.Lock()
defer mu.Unlock()
i.mu.Lock()
defer i.mu.Unlock()
i.translateStores[partitionID] = store
return nil
})

4
license.exceptions
View file

@ -11,4 +11,6 @@
./logger/filewriter_test.go
./vprint.go
./rbf/vprint.go
./cmd/loader/vprint.go
./cmd/slurp/vprint.go
./cmd/demo-lmdb/vprint.go
./gid.go

1800
lmdb/lmdb.go Normal file
View file

File diff suppressed because it is too large Load diff

1315
lmdb/lmdb_test.go Normal file
View file

File diff suppressed because it is too large Load diff

524
lmdb/txpool.go Normal file
View file

@ -0,0 +1,524 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build skip_building_lmdb_for_now
package pilosa
import (
"fmt"
"io"
"github.com/pilosa/pilosa/v2/roaring"
)
// poolTx directs all Tx calls to a pre-made
// goroutine pool that are setup to do
// LMDB operations safely. Each of these
// goroutines has had runtime.LockOSThread()
// called, and we serialize write Tx onto
// a single writer goroutine.
type poolTx struct {
w *LMDBWrapper
b *LMDBTx
}
var _ = (*LMDBWrapper).newPoolTx // happy linter
func (w *LMDBWrapper) newPoolTx(write bool, initialIndexName string) (ptx *poolTx) {
var tx *LMDBTx
job := newLMDBJob(write, func(j *lmdbJob) {
tx = w.NewLMDBTx(write, initialIndexName)
})
if suberr := w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return &poolTx{
w: w,
b: tx,
}
}
var _ Tx = (*poolTx)(nil)
func (c *poolTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
c.b.IncrementOpN(index, field, view, shard, changedN)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
}
func (c *poolTx) NewTxIterator(index, field, view string, shard uint64) (rit *roaring.Iterator) {
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
rit = c.b.NewTxIterator(index, field, view, shard)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
changed, rowSet, err = c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Dump() {
c.b.Dump()
}
func (c *poolTx) Readonly() bool {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Readonly() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Readonly()
}
func (tx *poolTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
func (c *poolTx) Rollback() {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
c.b.Rollback()
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
}
func (c *poolTx) Commit() (err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
err = c.b.Commit()
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) RoaringBitmap(index, field, view string, shard uint64) (rbm *roaring.Bitmap, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
rbm, err = c.b.RoaringBitmap(index, field, view, shard)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
ct, err = c.b.Container(index, field, view, shard, key)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) (err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
err = c.b.PutContainer(index, field, view, shard, key, rc)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) RemoveContainer(index, field, view string, shard uint64, key uint64) (err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
err = c.b.RemoveContainer(index, field, view, shard, key)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) UseRowCache() bool {
return c.b.UseRowCache()
}
func (c *poolTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Add() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
changeCount, err = c.b.Add(index, field, view, shard, batched, a...)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
changeCount, err = c.b.Remove(index, field, view, shard, a...)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
exists, err = c.b.Contains(index, field, view, shard, key)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
citer, found, err = c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) (err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
err = c.b.ForEach(index, field, view, shard, fn)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) (err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
err = c.b.ForEachRange(index, field, view, shard, start, end, fn)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Count(index, field, view string, shard uint64) (n uint64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
n, err = c.b.Count(index, field, view, shard)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Max(index, field, view string, shard uint64) (n uint64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
n, err = c.b.Max(index, field, view, shard)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Min(index, field, view string, shard uint64) (m uint64, found bool, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
m, found, err = c.b.Min(index, field, view, shard)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) (err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
err = c.b.UnionInPlace(index, field, view, shard, others...)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
n, err = c.b.CountRange(index, field, view, shard, start, end)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
other, err = c.b.OffsetRange(index, field, view, shard, offset, start, end)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
r, sz, err = c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}
func (c *poolTx) Type() string {
return c.b.Type()
}
func (c *poolTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
job := newLMDBJob(c.b.write, func(j *lmdbJob) {
sliceOfShards, err = c.b.SliceOfShards(index, field, view, optionalViewPath)
})
if suberr := c.w.submit(job); suberr != nil {
AlwaysPrintf("submit job saw err '%v'", suberr)
return
}
<-job.done
return
}

3
mmap_test.go
View file

@ -86,7 +86,8 @@ func forceSnapshotsCheckMapping(t *testing.T) {
// in newGeneration in generation.go. So this is probably useless but it's
// a failure mode we've been bitten by once...
func TestMmapBehavior(t *testing.T) {
skipForRBF(t)
// rbf and lmdb not happy with this test.
roaringOnlyTest(t)
var changed bool
var original uint64

47
pprof.go Normal file
View file

@ -0,0 +1,47 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"os"
"time"
_ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server.
"runtime/pprof"
)
func CPUProfileForDur(dur time.Duration, outpath string) {
// per-query pprof output:
txsrc := os.Getenv("PILOSA_TXSRC")
if txsrc == "" {
txsrc = "roaring"
}
path := outpath + "." + txsrc
f, err := os.Create(path)
panicOn(err)
if dur == 0 {
dur = time.Hour
}
vv("starting cpu profile for dur '%v', output to '%v'", dur, path)
_ = pprof.StartCPUProfile(f)
go func() {
<-time.After(dur)
pprof.StopCPUProfile()
f.Close()
vv("stopping cpu profile after dur '%v', output: '%v'", dur, path)
}()
}

380
rbf.go Normal file
View file

@ -0,0 +1,380 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"strings"
"sync"
"github.com/pilosa/pilosa/v2/rbf"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txpath"
"github.com/pkg/errors"
)
// RbfDBWrapper wraps an *rbf.DB
type RbfDBWrapper struct {
Path string
db *rbf.DB
reg *rbfDBRegistrar
muDb sync.Mutex
// make Close() idempotent, avoiding panic on double Close()
closed bool
//DeleteEmptyContainer bool // needed for roaring compat?
}
// rbfDBRegistrar also allows opening the same path twice to
// result in sharing the same open database handle, and
// thus the same transactional guarantees.
//
type rbfDBRegistrar struct {
mu sync.Mutex
mp map[*RbfDBWrapper]bool
path2db map[string]*RbfDBWrapper
}
var globalRbfDBReg *rbfDBRegistrar = newRbfDBRegistrar()
func newRbfDBRegistrar() *rbfDBRegistrar {
return &rbfDBRegistrar{
mp: make(map[*RbfDBWrapper]bool),
path2db: make(map[string]*RbfDBWrapper),
}
}
// register each rbf.DB created, so we dedup and can
// can clean them up. This is called by openRbfDB() while
// holding the r.mu.Lock, since it needs to atomically
// check the registry and make a new instance only
// if one does not exist for its path, and otherwise
// return the existing instance.
func (r *rbfDBRegistrar) unprotectedRegister(w *RbfDBWrapper) {
r.mp[w] = true
r.path2db[w.Path] = w
}
// unregister removes w from r
func (r *rbfDBRegistrar) unregister(w *RbfDBWrapper) {
r.mu.Lock()
delete(r.mp, w)
delete(r.path2db, w.Path)
r.mu.Unlock()
}
// rbfPath is a helper for determining the full directory
// in which the RBF database will be stored.
func rbfPath(path string) string {
if !strings.HasSuffix(path, "-rbfdb") {
return path + "-rbfdb"
}
return path
}
// openRbfDB opens the database in the path directoy
// without deleting any prior content. Any
// database directory will have the "-rbfdb" suffix.
//
// openRbfDB will check the registry and make a new instance only
// if one does not exist for its path. Otherwise it returns
// the existing instance. This insures only one RbfDBWrapper
// per bpath in this pilosa node.
func (r *rbfDBRegistrar) openRbfDB(path0 string) (*RbfDBWrapper, error) {
path := rbfPath(path0)
r.mu.Lock()
defer r.mu.Unlock()
w, ok := r.path2db[path]
if ok {
// creates the effect of having only one DB open per pilosa node.
return w, nil
}
db := rbf.NewDB(path)
w = &RbfDBWrapper{
reg: r,
Path: path,
db: db,
}
r.unprotectedRegister(w)
err := db.Open()
if err != nil {
panic(fmt.Sprintf("cannot open rbfDB at path '%v': '%v'", path, err))
}
return w, nil
}
type RBFTx struct {
// initialIndex is only a debugging aid. Transactions
// can cross indexes. It can be left empty without consequence.
initialIndex string
frag *fragment
tx *rbf.Tx
}
func (tx *RBFTx) DBPath() string {
return tx.tx.DBPath()
}
func (tx *RBFTx) Type() string {
return RBFTxn
}
func (tx *RBFTx) Rollback() {
tx.tx.Rollback()
}
func (tx *RBFTx) Commit() error {
return tx.tx.Commit()
}
func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
return tx.tx.RoaringBitmap(rbfName(index, field, view, shard))
}
func (tx *RBFTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
return tx.tx.Container(rbfName(index, field, view, shard), key)
}
func (tx *RBFTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
return tx.tx.PutContainer(rbfName(index, field, view, shard), key, c)
}
func (tx *RBFTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
return tx.tx.RemoveContainer(rbfName(index, field, view, shard), key)
}
func (tx *RBFTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
return tx.tx.Add(rbfName(index, field, view, shard), a...)
}
func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
return tx.tx.Remove(rbfName(index, field, view, shard), a...)
}
func (tx *RBFTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
return tx.tx.Contains(rbfName(index, field, view, shard), v)
}
func (tx *RBFTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
return tx.tx.ContainerIterator(rbfName(index, field, view, shard), key)
}
func (tx *RBFTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
return tx.tx.ForEach(rbfName(index, field, view, shard), fn)
}
func (tx *RBFTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
return tx.tx.ForEachRange(rbfName(index, field, view, shard), start, end, fn)
}
func (tx *RBFTx) Count(index, field, view string, shard uint64) (uint64, error) {
return tx.tx.Count(rbfName(index, field, view, shard))
}
func (tx *RBFTx) Max(index, field, view string, shard uint64) (uint64, error) {
return tx.tx.Max(rbfName(index, field, view, shard))
}
func (tx *RBFTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
return tx.tx.Min(rbfName(index, field, view, shard))
}
func (tx *RBFTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
return tx.tx.UnionInPlace(rbfName(index, field, view, shard), others...)
}
// CountRange returns the count of hot bits in the start, end range on the fragment.
// roaring.countRange counts the number of bits set between [start, end).
func (tx *RBFTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
if tx.frag == nil {
return tx.tx.CountRange(rbfName(index, field, view, shard), start, end)
}
// For speed, exploit the fact that on startup the rowCache will
// have already loaded fragments.
rowID := start / ShardWidth
row, err := tx.frag.unprotectedRow(tx, rowID)
if err != nil {
return 0, err
}
return row.Count(), nil
}
func (tx *RBFTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
return tx.tx.OffsetRange(rbfName(index, field, view, shard), offset, start, end)
}
func (tx *RBFTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {}
func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
return tx.tx.ImportRoaringBits(rbfName(index, field, view, shard), rit, clear, log, rowSize, data)
}
func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
rbm, err := tx.RoaringBitmap(index, field, view, shard)
if err != nil {
return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap")
}
var buf bytes.Buffer
sz, err = rbm.WriteTo(&buf)
if err != nil {
return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)")
}
return ioutil.NopCloser(&buf), sz, err
}
func (tx *RBFTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
prefix := string(txpath.AllShardPrefix(index, field, view))
names, err := tx.tx.BitmapNames()
if err != nil {
return nil, err
}
// Iterate over shard names and collect shards from matching field/view prefix.
for _, name := range names {
if !strings.HasPrefix(name, prefix) {
continue
}
shard := txpath.ShardFromPrefix([]byte(name))
sliceOfShards = append(sliceOfShards, shard)
}
return sliceOfShards, nil
}
func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
b, err := tx.RoaringBitmap(index, field, view, shard)
panicOn(err)
return b.Iterator()
}
func (tx *RBFTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
func (tx *RBFTx) Dump() {
tx.tx.Dump()
}
// Readonly is true if the transaction is not read-and-write, but only doing reads.
func (tx *RBFTx) Readonly() bool {
return !tx.tx.Writable()
}
func (tx *RBFTx) UseRowCache() bool {
// since RFB returns memory mapped data, we can't use
// the rowCache without first making a copy.
// So we only use the rowCache if the copy is
// enabled.
return rbf.EnableRowCache
}
// rbfName returns a NULL-separated key used for identifying bitmap maps in RBF.
func rbfName(index, field, view string, shard uint64) string {
//return fmt.Sprintf("%s\x00%s\x00%s\x00%d", index, field, view, shard)
return string(txpath.Prefix(index, field, view, shard))
}
// rbfFieldPrefix returns a prefix for field keys in RBF.
func rbfFieldPrefix(index, field string) string {
//return fmt.Sprintf("%s\x00%s\x00", index, field)
return string(txpath.FieldPrefix(index, field))
}
func (w *RbfDBWrapper) DeleteField(index, field, fieldPath string) error {
w.muDb.Lock()
defer w.muDb.Unlock()
if err := os.RemoveAll(fieldPath); err != nil {
return errors.Wrap(err, "removing directory")
}
tx, err := w.db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
if err := tx.DeleteBitmapsWithPrefix(rbfFieldPrefix(index, field)); err != nil {
return err
}
return tx.Commit()
}
func (w *RbfDBWrapper) DeleteIndex(indexName string) error {
if strings.Contains(indexName, "'") {
return fmt.Errorf("error: bad indexName `%v` in RbfDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes.", indexName)
}
prefix := txpath.IndexOnlyPrefix(indexName)
w.muDb.Lock()
defer w.muDb.Unlock()
tx, err := w.db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
if err := tx.DeleteBitmapsWithPrefix(string(prefix)); err != nil {
return err
}
return tx.Commit()
}
func (w *RbfDBWrapper) Close() error {
w.muDb.Lock()
defer w.muDb.Unlock()
if !w.closed {
w.reg.unregister(w)
w.closed = true
}
return w.db.Close()
}
func (w *RbfDBWrapper) NewRBFTx(write bool, initialIndex string, frag *fragment) (*RBFTx, error) {
tx, err := w.db.Begin(write)
if err != nil {
return nil, err
}
return &RBFTx{tx: tx, initialIndex: initialIndex, frag: frag}, nil
}
func (w *RbfDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag *fragment) error {
tx, err := w.db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
err = tx.DeleteBitmapsWithPrefix(rbfName(index, field, view, shard))
if err != nil {
return err
}
return tx.Commit()
}

17
rbf/cursor.go
View file

@ -371,7 +371,8 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) {
// Split into multiple pages if page size is exceeded.
groups := [][]leafCell{cells}
if leafCellsPageSize(cells) >= PageSize {
sz := leafCellsPageSize(cells)
if sz >= PageSize {
groups = splitLeafCells(cells)
}
@ -674,6 +675,9 @@ func splitLeafCells(cells []leafCell) [][]leafCell {
var dataSize int
for _, cell := range cells {
if cell.Type == ContainerTypeBitmap {
panic("no! all ContainerTypeBitmap should be ContainerTypeBitmapPtr by now")
}
// Determine number of cells on current slice & cell size.
cellN := len(slices[len(slices)-1])
sz := align8(leafCellHeaderSize + len(cell.Data))
@ -823,7 +827,7 @@ func (c *Cursor) Seek(key uint64) (exact bool, err error) {
switch typ := readFlags(buf); typ {
case PageTypeBranch:
n := readCellN(buf)
index, ok := search(n, func(i int) int {
index, xact := search(n, func(i int) int {
if v := readBranchCellKey(buf, i); key == v {
return 0
} else if key < v {
@ -831,8 +835,8 @@ func (c *Cursor) Seek(key uint64) (exact bool, err error) {
}
return 1
})
//if not found (ok) the cell
if !ok && index > 0 {
//if not found (xact) the cell
if !xact && index > 0 {
index--
}
elem.index = index
@ -848,7 +852,7 @@ func (c *Cursor) Seek(key uint64) (exact bool, err error) {
case PageTypeLeaf:
n := readCellN(buf)
index, ok := search(n, func(i int) int {
index, xact := search(n, func(i int) int {
if v := readLeafCellKey(buf, i); key == v {
return 0
} else if key < v {
@ -858,7 +862,7 @@ func (c *Cursor) Seek(key uint64) (exact bool, err error) {
})
elem.index = index
c.leafPage = buf
return ok, nil
return xact, nil
default:
return false, fmt.Errorf("rbf.Cursor.Seek(): invalid page type: pgno=%d type=%d", elem.pgno, typ)
@ -1132,6 +1136,7 @@ func ConvertToLeafArgs(key uint64, c *roaring.Container) (result leafCell) {
roaring.ConvertRunToBitmap(c)
result.Type = ContainerTypeBitmap
result.Data = fromArray64(roaring.AsBitmap(c))
return
}
result.N = len(r) //note RBF N is number of containers
result.Type = ContainerTypeRLE

33
rbf/cursorx.go
View file

@ -24,6 +24,10 @@ import (
"github.com/pkg/errors"
)
// if enableRowCache, then we must not return mmap-ed memory
// directly, but only a copy.
const EnableRowCache = true
//probably should just implement the container interface
// but for now i'll do it
func (c *Cursor) Rows() ([]uint64, error) {
@ -53,7 +57,7 @@ func (c *Cursor) Rows() ([]uint64, error) {
return rows, err
}
func (tx *Tx) FieldViews() []string {
r, _ := tx.rootRecords()
r, _ := tx.RootRecords()
res := make([]string, len(r))
for i := range r {
res[i] = r[i].Name
@ -140,16 +144,33 @@ func (c *Cursor) CurrentPageType() int {
}
func toContainer(l leafCell, tx *Tx) *roaring.Container {
orig := l.Data
var cpMaybe []byte
if EnableRowCache {
// make a copy, otherwise the rowCache will see corrupted data
// or mmapped data that may disappear.
cpMaybe = make([]byte, len(orig))
copy(cpMaybe, orig)
} else {
// not a copy
cpMaybe = orig
}
switch l.Type {
case ContainerTypeArray:
return roaring.NewContainerArray(toArray16(l.Data))
return roaring.NewContainerArray(toArray16(cpMaybe))
case ContainerTypeBitmapPtr:
_, bm, _ := tx.leafCellBitmap(toPgno(l.Data))
return roaring.NewContainerBitmap(l.N, bm)
_, bm, _ := tx.leafCellBitmap(toPgno(cpMaybe))
cloneMaybe := bm
if EnableRowCache {
cloneMaybe = make([]uint64, len(bm))
copy(cloneMaybe, bm)
}
return roaring.NewContainerBitmap(l.N, cloneMaybe)
case ContainerTypeBitmap:
return roaring.NewContainerBitmap(l.N, toArray64(l.Data))
return roaring.NewContainerBitmap(l.N, toArray64(cpMaybe))
case ContainerTypeRLE:
return roaring.NewContainerRun(toInterval16(l.Data))
return roaring.NewContainerRun(toInterval16(cpMaybe))
}
return nil
}

55
rbf/db.go
View file

@ -39,12 +39,13 @@ const (
)
type DB struct {
data []byte // mmap data
file *os.File // file descriptor
segments []*WALSegment // write-ahead log
pageMap *immutable.Map // pgno-to-WALID mapping
txs map[*Tx]struct{} // active transactions
opened bool // true if open
data []byte // mmap data
file *os.File // file descriptor
segments []*WALSegment // write-ahead log
rootRecords []*RootRecord // cached root records
pageMap *immutable.Map // pgno-to-WALID mapping
txs map[*Tx]struct{} // active transactions
opened bool // true if open
mu sync.RWMutex // general mutex
rwmu sync.Mutex // mutex for restricting single writer
@ -58,12 +59,13 @@ type DB struct {
// NewDB returns a new instance of DB.
func NewDB(path string) *DB {
return &DB{
db := &DB{
txs: make(map[*Tx]struct{}),
pageMap: immutable.NewMap(&uint32Hasher{}),
Path: path,
MaxSize: DefaultMaxSize,
}
return db
}
// DataPath returns the path to the data file for the DB.
@ -73,10 +75,12 @@ func (db *DB) DataPath() string {
// WALPath returns the path to the WAL directory.
func (db *DB) WALPath() string {
return filepath.Join(db.Path, "wal")
}
func CreateDirIfNotExist(path string) {
dir := filepath.Dir(path)
if _, err := os.Stat(dir); os.IsNotExist(err) {
err = os.MkdirAll(dir, 0755)
@ -89,6 +93,7 @@ func CreateDirIfNotExist(path string) {
// Open opens a database with the file specified in Path.
// Creates a new file if one does not already exist.
func (db *DB) Open() (err error) {
db.mu.Lock()
defer db.mu.Unlock()
@ -137,6 +142,7 @@ func (db *DB) Open() (err error) {
}
func (db *DB) openWALSegments() error {
fis, err := ioutil.ReadDir(db.WALPath())
if err != nil {
return fmt.Errorf("read dir: %w", err)
@ -170,6 +176,7 @@ func (db *DB) openWALSegments() error {
// only copy pages that aren't in use by an active transaction. The page map
// is rebuilt as well for all WAL pages still in use.
func (db *DB) checkpoint() error {
if !db.opened {
return nil
}
@ -257,8 +264,11 @@ func (db *DB) checkpoint() error {
break
}
segpath := segment.Path()
if err := segment.Close(); err != nil {
return err
} else if err := os.Remove(segpath); err != nil {
return err
}
db.segments, db.segments[0] = db.segments[1:], nil
}
@ -269,6 +279,7 @@ func (db *DB) checkpoint() error {
}
func (db *DB) findNextWALMetaPage(walID int64) (metaWALID int64, metaFlags uint32, err error) {
maxWALID := db.maxWALID()
for ; walID <= maxWALID; walID++ {
@ -292,6 +303,7 @@ func (db *DB) findNextWALMetaPage(walID int64) (metaWALID int64, metaFlags uint3
// minActiveWALID returns the lowest WAL ID in use by any active transaction.
// Returns 0 if no transactions are active.
func (db *DB) minActiveWALID() int64 {
var walID int64
for tx := range db.txs {
if walID == 0 || walID > tx.walID {
@ -303,12 +315,14 @@ func (db *DB) minActiveWALID() int64 {
// ActiveWALSegment returns the most recent WAL segment.
func (db *DB) ActiveWALSegment() *WALSegment {
db.mu.RLock()
defer db.mu.RUnlock()
return db.activeWALSegment()
}
func (db *DB) activeWALSegment() *WALSegment {
if len(db.segments) == 0 {
return nil
}
@ -317,12 +331,14 @@ func (db *DB) activeWALSegment() *WALSegment {
// MinWALID returns the lowest WAL ID available in the WAL.
func (db *DB) MinWALID() int64 {
db.mu.RLock()
defer db.mu.RUnlock()
return db.minWALID()
}
func (db *DB) minWALID() int64 {
if len(db.segments) == 0 {
return 0
}
@ -331,12 +347,14 @@ func (db *DB) minWALID() int64 {
// MaxWALID returns the highest WAL ID available in the WAL.
func (db *DB) MaxWALID() int64 {
db.mu.RLock()
defer db.mu.RUnlock()
return db.maxWALID()
}
func (db *DB) maxWALID() int64 {
if len(db.segments) == 0 {
return 0
}
@ -346,6 +364,7 @@ func (db *DB) maxWALID() int64 {
// WALPageN returns the number of pages across all segments.
func (db *DB) WALPageN() int64 {
db.mu.RLock()
defer db.mu.RUnlock()
@ -358,6 +377,7 @@ func (db *DB) WALPageN() int64 {
// SyncWAL flushes the active segment to disk.
func (db *DB) SyncWAL() error {
if s := db.ActiveWALSegment(); s != nil {
return s.Sync()
}
@ -366,6 +386,8 @@ func (db *DB) SyncWAL() error {
// readWALPage reads a single page at the given WAL ID.
func (db *DB) readWALPage(walID int64) ([]byte, error) {
//
// TODO(BBJ): Binary search for segment.
for _, s := range db.segments {
if walID >= s.MinWALID() && walID <= s.MaxWALID() {
@ -376,6 +398,7 @@ func (db *DB) readWALPage(walID int64) ([]byte, error) {
}
func (db *DB) writeWALPage(page []byte, isMeta bool) (walID int64, err error) {
if err := db.ensureWritableWALSegment(); err != nil {
return 0, err
}
@ -383,6 +406,7 @@ func (db *DB) writeWALPage(page []byte, isMeta bool) (walID int64, err error) {
}
func (db *DB) writeBitmapPage(pgno uint32, page []byte) (walID int64, err error) {
if err := db.ensureWritableWALSegment(); err != nil {
return 0, err
}
@ -401,6 +425,7 @@ func (db *DB) writeBitmapPage(pgno uint32, page []byte) (walID int64, err error)
}
func (db *DB) ensureWritableWALSegment() error {
if s := db.activeWALSegment(); s != nil && s.Size() < MaxWALSegmentFileSize {
return nil
}
@ -409,6 +434,7 @@ func (db *DB) ensureWritableWALSegment() error {
// addWALSegment appends a new, writable segment and closing an existing segments for write.
func (db *DB) addWALSegment() error {
// Close previous last segment for writes.
base := int64(1)
if s := db.activeWALSegment(); s != nil {
@ -430,6 +456,7 @@ func (db *DB) addWALSegment() error {
// Close closes the database.
func (db *DB) Close() (err error) {
// TODO(bbj): Add wait group to hang until last Tx is complete.
// Wait for writer lock.
@ -466,6 +493,7 @@ func (db *DB) Close() (err error) {
// closeWALSegments closes the WAL and all its segments.
func (db *DB) closeWALSegments() (err error) {
for _, s := range db.segments {
if e := s.Close(); e != nil && err == nil {
err = e
@ -476,6 +504,7 @@ func (db *DB) closeWALSegments() (err error) {
// Size returns the size of the database & WAL, in bytes.
func (db *DB) Size() (int64, error) {
db.mu.RLock()
defer db.mu.RUnlock()
@ -488,12 +517,14 @@ func (db *DB) Size() (int64, error) {
// WALSize returns the size of all WAL segments, in bytes.
func (db *DB) WALSize() int64 {
db.mu.RLock()
defer db.mu.RUnlock()
return db.walSize()
}
func (db *DB) walSize() int64 {
var sz int64
for _, s := range db.segments {
sz += s.Size()
@ -504,6 +535,7 @@ func (db *DB) walSize() int64 {
// WALSegments returns the WAL segments currently on the DB.
// This should only be used for debugging & testing purposes.
func (db *DB) WALSegments() []*WALSegment {
db.mu.RLock()
defer db.mu.RUnlock()
return db.segments
@ -511,6 +543,7 @@ func (db *DB) WALSegments() []*WALSegment {
// init initializes a new database file.
func (db *DB) init() error {
if err := db.initMetaPage(); err != nil {
return fmt.Errorf("meta: %w", err)
} else if err := db.initRootRecordPage(); err != nil {
@ -523,6 +556,7 @@ func (db *DB) init() error {
// initMetaPage initializes the meta page.
func (db *DB) initMetaPage() error {
page := make([]byte, PageSize)
writeMetaMagic(page)
writeMetaPageN(page, 3)
@ -534,6 +568,7 @@ func (db *DB) initMetaPage() error {
// initRootRecordPage initializes the initial root record page.
func (db *DB) initRootRecordPage() error {
page := make([]byte, PageSize)
writePageNo(page, 1)
writeFlags(page, PageTypeRootRecord)
@ -543,6 +578,7 @@ func (db *DB) initRootRecordPage() error {
// initFreelistPage initializes the initial freelist btree page.
func (db *DB) initFreelistPage() error {
page := make([]byte, PageSize)
writePageNo(page, 2)
writeFlags(page, PageTypeLeaf)
@ -552,6 +588,7 @@ func (db *DB) initFreelistPage() error {
// Begin starts a new transaction.
func (db *DB) Begin(writable bool) (_ *Tx, err error) {
// TODO(BBJ): Acquire write lock if writable.
// Ensure only one writable transaction at a time.
@ -566,7 +603,7 @@ func (db *DB) Begin(writable bool) (_ *Tx, err error) {
return nil, ErrClosed
}
tx := &Tx{db: db, pageMap: db.pageMap, writable: writable}
tx := &Tx{db: db, rootRecords: db.rootRecords, pageMap: db.pageMap, writable: writable}
// Copy meta page into transaction's buffer.
// This page is only written at the end of a dirty transaction.
@ -614,6 +651,7 @@ func (db *DB) removeTx(tx *Tx) error {
// Check performs an integrity check.
func (db *DB) Check() error {
tx, err := db.Begin(false)
if err != nil {
return err
@ -624,6 +662,7 @@ func (db *DB) Check() error {
// writePage writes a page to the data file.
func (db *DB) writePage(pgno uint32, page []byte) error {
_, err := db.file.WriteAt(page, int64(pgno)*PageSize)
return err
}

39
rbf/rbf.go
View file

@ -351,7 +351,7 @@ func (c *leafCell) Values(tx *Tx) []uint16 {
}
// firstValue the first value from the container.
func (c *leafCell) firstValue() uint16 {
func (c *leafCell) firstValue(tx *Tx) uint16 {
switch c.Type {
case ContainerTypeArray:
a := toArray16(c.Data)
@ -360,7 +360,9 @@ func (c *leafCell) firstValue() uint16 {
r := toInterval16(c.Data)
return r[0].Start
case ContainerTypeBitmapPtr:
for i, v := range toArray64(c.Data) {
_, slc, err := tx.leafCellBitmap(toPgno(c.Data))
panicOn(err)
for i, v := range slc {
for j := uint(0); j < 64; j++ {
if v&(1<<j) != 0 {
return (uint16(i) * 64) + uint16(j)
@ -373,8 +375,20 @@ func (c *leafCell) firstValue() uint16 {
}
}
// helper for lastValue()
func (c *leafCell) lastValueFromBitmap(a []uint64) uint16 {
for i := len(a) - 1; i >= 0; i-- {
for j := 63; j >= 0; j-- {
if a[i]&(1<<j) != 0 {
return (uint16(i) * 64) + uint16(j)
}
}
}
panic(fmt.Sprintf("rbf.leafCell.lastValueFromBitmap(): no values set in bitmap container: key=%d", c.Key))
}
// lastValue the last value from the container.
func (c *leafCell) lastValue() uint16 {
func (c *leafCell) lastValue(tx *Tx) uint16 {
switch c.Type {
case ContainerTypeArray:
a := toArray16(c.Data)
@ -382,16 +396,15 @@ func (c *leafCell) lastValue() uint16 {
case ContainerTypeRLE:
r := toInterval16(c.Data)
return r[len(r)-1].Last
case ContainerTypeBitmap:
a := toArray64(c.Data)
for i := len(a) - 1; i >= 0; i-- {
for j := 63; j >= 0; j-- {
if a[i]&(1<<j) != 0 {
return (uint16(i) * 64) + uint16(j)
}
}
}
panic(fmt.Sprintf("rbf.leafCell.firstValue(): no values set in bitmap container: key=%d", c.Key))
return c.lastValueFromBitmap(a)
case ContainerTypeBitmapPtr:
_, a, err := tx.leafCellBitmap(toPgno(c.Data))
panicOn(err)
return c.lastValueFromBitmap(a)
default:
panic(fmt.Sprintf("invalid container type: %d", c.Type))
}
@ -429,8 +442,6 @@ func readLeafCell(page []byte, i int) leafCell {
assert(i < readCellN(page), "cell index %d exceeds cell count %d", i, readCellN(page))
offset := readCellOffset(page, i)
// cd ..; PILOSA_TXSRC=rbf go test -v -run TestFragment_TopN_IDs -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0"
// gives panic: runtime error: slice bounds out of range [16390:8192] here.
buf := page[offset:]
var cell leafCell
@ -476,7 +487,7 @@ func writeLeafCell(page []byte, i, offset int, cell leafCell) {
*(*uint32)(unsafe.Pointer(&page[offset+8])) = uint32(cell.Type)
*(*uint16)(unsafe.Pointer(&page[offset+12])) = uint16(cell.N)
*(*uint16)(unsafe.Pointer(&page[offset+14])) = uint16(cell.BitN)
assert(offset+16+len(cell.Data) <= PageSize, "leaf cell write extends beyond page: offset %d + cell size %d > page size %d", offset, 16+len(cell.Data), PageSize)
assert(offset+16+len(cell.Data) <= PageSize, "leaf cell write extends beyond page: offset %d + len(cell.Data)(%v) + 16 == %v > page size %d", offset, len(cell.Data), offset+16+len(cell.Data), PageSize)
copy(page[offset+16:], cell.Data)
}

208
rbf/tx.go
View file

@ -14,12 +14,11 @@
package rbf
import (
"bytes"
"fmt"
"io"
"math"
"sort"
"strconv"
//"strconv"
"strings"
"sync"
@ -29,13 +28,14 @@ import (
// Tx represents a transaction.
type Tx struct {
mu sync.RWMutex
db *DB // parent db
meta [PageSize]byte // copy of current meta page
walID int64 // max WAL ID at start of tx
pageMap *immutable.Map // mapping of database pages to WAL IDs
writable bool // if true, tx can write
dirty bool // if true, changes have been made
mu sync.RWMutex
db *DB // parent db
meta [PageSize]byte // copy of current meta page
walID int64 // max WAL ID at start of tx
rootRecords []*RootRecord // read-only cache of root records
pageMap *immutable.Map // mapping of database pages to WAL IDs
writable bool // if true, tx can write
dirty bool // if true, changes have been made
// If Rollback() has already completed, don't do it again.
// Note db == nil means that commit has already been done.
@ -47,6 +47,10 @@ type Tx struct {
DeleteEmptyContainer bool
}
func (tx *Tx) DBPath() string {
return tx.db.Path
}
// Writable returns true if the transaction can mutate data.
func (tx *Tx) Writable() bool {
return tx.writable
@ -69,7 +73,17 @@ func (tx *Tx) Commit() error {
} else if err := tx.db.SyncWAL(); err != nil {
return err
}
// future plan: after checkpoint is moved to background
// or not every removeTx, then we can move the
// tx.db.rootRecords = tx.rootRecords into removeTx().
// avoid race detector firing on a write race here
// vs the read of rootRecords at db.Begin()
tx.db.mu.Lock()
tx.db.rootRecords = tx.rootRecords
tx.db.pageMap = tx.pageMap
tx.db.mu.Unlock()
}
if err := tx.db.checkpoint(); err != nil {
@ -83,6 +97,7 @@ func (tx *Tx) Commit() error {
func (tx *Tx) Rollback() {
tx.mu.Lock()
defer tx.mu.Unlock()
// allow Rollback to be called more than once.
if tx.rollbackDone {
return
@ -118,7 +133,7 @@ func (tx *Tx) Root(name string) (uint32, error) {
}
func (tx *Tx) root(name string) (uint32, error) {
records, err := tx.rootRecords()
records, err := tx.RootRecords()
if err != nil {
return 0, err
}
@ -140,7 +155,7 @@ func (tx *Tx) BitmapNames() ([]string, error) {
}
// Read list of root records.
records, err := tx.rootRecords()
records, err := tx.RootRecords()
if err != nil {
return nil, err
}
@ -162,8 +177,6 @@ func (tx *Tx) CreateBitmap(name string) error {
}
func (tx *Tx) createBitmap(name string) error {
//vv("createBitmap(name='%v'", name)
if tx.db == nil {
return ErrTxClosed
} else if !tx.writable {
@ -173,7 +186,7 @@ func (tx *Tx) createBitmap(name string) error {
}
// Read list of root records.
records, err := tx.rootRecords()
records, err := tx.RootRecords()
if err != nil {
return err
}
@ -252,7 +265,7 @@ func (tx *Tx) DeleteBitmap(name string) error {
}
// Read list of root records.
records, err := tx.rootRecords()
records, err := tx.RootRecords()
if err != nil {
return err
}
@ -274,7 +287,7 @@ func (tx *Tx) DeleteBitmap(name string) error {
if err := tx.writeRootRecordPages(records); err != nil {
return fmt.Errorf("write bitmaps: %w", err)
}
tx.rootRecords = records
return nil
}
@ -290,7 +303,7 @@ func (tx *Tx) DeleteBitmapsWithPrefix(prefix string) error {
}
// Read list of root records.
records, err := tx.rootRecords()
records, err := tx.RootRecords()
if err != nil {
return err
}
@ -317,7 +330,7 @@ func (tx *Tx) DeleteBitmapsWithPrefix(prefix string) error {
if err := tx.writeRootRecordPages(records); err != nil {
return fmt.Errorf("write bitmaps: %w", err)
}
tx.rootRecords = records
return nil
}
@ -336,7 +349,7 @@ func (tx *Tx) RenameBitmap(oldname, newname string) error {
}
// Read list of root records.
records, err := tx.rootRecords()
records, err := tx.RootRecords()
if err != nil {
return err
}
@ -356,8 +369,12 @@ func (tx *Tx) RenameBitmap(oldname, newname string) error {
return nil
}
// rootRecords returns a list of root records.
func (tx *Tx) rootRecords() ([]*RootRecord, error) {
// RootRecords returns a list of root records.
func (tx *Tx) RootRecords() (rr []*RootRecord, err error) {
if tx.rootRecords != nil {
return tx.rootRecords, nil
}
var records []*RootRecord
for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; {
page, err := tx.readPage(pgno)
@ -375,11 +392,15 @@ func (tx *Tx) rootRecords() ([]*RootRecord, error) {
// Read next overflow page number.
pgno = WalkRootRecordPages(page)
}
// Cache result
tx.rootRecords = records
return records, nil
}
// writeRootRecordPages writes a list of root record pages.
func (tx *Tx) writeRootRecordPages(records []*RootRecord) (err error) {
// Release all existing root record pages.
for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; {
page, err := tx.readPage(pgno)
@ -434,13 +455,14 @@ func (tx *Tx) writeRootRecordPages(records []*RootRecord) (err error) {
}
}
// Update cache records.
tx.rootRecords = records
return nil
}
// Add sets a given bit on the bitmap.
func (tx *Tx) Add(name string, a ...uint64) (changeCount int, err error) {
//vv("rbf Tx.Add(a='%#v')", a)
tx.mu.Lock()
defer tx.mu.Unlock()
@ -609,9 +631,10 @@ func (tx *Tx) PutContainer(name string, key uint64, ct *roaring.Container) error
tx.mu.Lock()
defer tx.mu.Unlock()
if ct.N() == 0 {
return nil
if tx.DeleteEmptyContainer && ct.N() == 0 {
return tx.RemoveContainer(name, key)
}
cell := ConvertToLeafArgs(key, ct)
if err := tx.createBitmapIfNotExists(name); err != nil {
@ -745,7 +768,7 @@ func (tx *Tx) inusePageSet() (map[uint32]struct{}, error) {
}
// Traverse every b-tree and mark pages as in-use.
records, err := tx.rootRecords()
records, err := tx.RootRecords()
if err != nil {
return m, err
}
@ -826,7 +849,7 @@ func (tx *Tx) nextFreelistPageNo() (uint32, error) {
}
cell := c.cell()
v := cell.firstValue()
v := cell.firstValue(tx)
pgno := uint32((cell.Key << 16) | uint64(v))
return pgno, nil
@ -870,7 +893,6 @@ func (tx *Tx) deallocateTree(pgno uint32) error {
}
func (tx *Tx) readPage(pgno uint32) ([]byte, error) {
// fmt.Println("readPage", pgno)
// Meta page is always cached on the transaction.
if pgno == 0 {
return tx.meta[:], nil
@ -964,10 +986,11 @@ func (tx *Tx) ContainerIterator(name string, key uint64) (citer roaring.Containe
// INVAR: c is not nil
if _, err := c.Seek(key); err != nil {
exact, err := c.Seek(key)
if err != nil {
return nil, false, err
}
return &containerIterator{cursor: c}, true, nil
return &containerIterator{cursor: c}, exact, nil
}
func (tx *Tx) ForEach(name string, fn func(i uint64) error) error {
@ -1084,7 +1107,7 @@ func (tx *Tx) Max(name string) (uint64, error) {
}
cell := c.cell()
return uint64((cell.Key << 16) | uint64(cell.lastValue())), nil
return uint64((cell.Key << 16) | uint64(cell.lastValue(tx))), nil
}
func (tx *Tx) Min(name string) (uint64, bool, error) {
@ -1103,11 +1126,28 @@ func (tx *Tx) Min(name string) (uint64, bool, error) {
}
cell := c.cell()
return uint64((cell.Key << 16) | uint64(cell.firstValue())), true, nil
return uint64((cell.Key << 16) | uint64(cell.firstValue(tx))), true, nil
}
func (tx *Tx) UnionInPlace(name string, others ...*roaring.Bitmap) error {
panic("TODO")
rbm, err := tx.RoaringBitmap(name)
panicOn(err)
rbm.UnionInPlace(others...)
// iterate over the containers that changed within rbm, and write them back to disk.
it, found := rbm.Containers.Iterator(0)
_ = found // don't care about the value of found, because first containerKey might be > 0
for it.Next() {
containerKey, rc := it.Value()
// TODO: only write the changed ones back, as optimization?
// Compare to ImportRoaringBits.
err := tx.PutContainer(name, containerKey, rc)
panicOn(err)
}
return nil
}
// roaring.countRange counts the number of bits set between [start, end).
@ -1187,8 +1227,10 @@ func (tx *Tx) OffsetRange(name string, offset, start, endx uint64) (*roaring.Bit
panic("range endx must not contain low bits")
}
tx.mu.RLock()
defer tx.mu.RUnlock()
// need write lock here (not just read lock) b/c caching the tx.rootRecords = records
// is a write the race detector fires on.
tx.mu.Lock()
defer tx.mu.Unlock()
c, err := tx.cursor(name)
if err != nil {
@ -1261,15 +1303,15 @@ func (si *emptyContainerIterator) Value() (uint64, *roaring.Container) {
panic("emptyContainerIterator never has any Values")
}
func (tx *Tx) Dump(index string) {
fmt.Println(tx.DumpString(index))
func (tx *Tx) Dump() {
fmt.Println(tx.DumpString())
}
func (tx *Tx) DumpString(index string) (r string) {
func (tx *Tx) DumpString() (r string) {
r = "allkeys:[\n"
// grab root records, for a list of bitmaps.
records, err := tx.rootRecords()
records, err := tx.RootRecords()
panicOn(err)
n := 0
for _, rr := range records {
@ -1292,7 +1334,7 @@ func (tx *Tx) DumpString(index string) (r string) {
ckey := cell.Key
ct := toContainer(cell, tx)
s := stringOfCkeyCt(ckey, ct, rr.Name, index)
s := stringOfCkeyCt(ckey, ct, rr.Name)
r += s
n++
}
@ -1307,55 +1349,21 @@ func (tx *Tx) DumpString(index string) (r string) {
}
func containerToBytes(ct *roaring.Container) []byte {
ty := roaring.ContainerType(ct)
switch ty {
case containerNil:
case roaring.ContainerNil:
panic("nil container")
case containerArray:
case roaring.ContainerArray:
return fromArray16(roaring.AsArray(ct))
case containerBitmap:
case roaring.ContainerBitmap:
return fromArray64(roaring.AsBitmap(ct))
case containerRun:
case roaring.ContainerRun:
return fromInterval16(roaring.AsRuns(ct))
}
panic(fmt.Sprintf("unknown container type '%v'", int(ty)))
}
func badgerKey(index, field, view string, shard uint64, roaringContainerKey uint64) []byte {
// The %020d which adds zero padding up to 20 runes is required to
// allow the textual sort to accurately
// reflect a numeric sort order. This is because, as a string,
// math.MaxUint64 is 20 bytes long.
// Example of such a badgerKey with a container-key that is math.MaxUint64:
// ...........................................12345678901234567890
// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615
prefix := badgerPrefix(index, field, view, shard)
ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
bkey := append(prefix, ckey...)
MustValidateKey(bkey)
return bkey
}
// badgerPrefix returns everything from badgerKey up to and
// including the '@' fune in a badger key. The prefix excludes the roaring container key itself.
// NB must be kept in sync with badgerKey() and badgerKeyExtractContainerKey().
func badgerPrefix(index, field, view string, shard uint64) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@", index, field, view, shard))
}
// MustValidatekey will panic on a bad badgerKey with an informative message.
func MustValidateKey(bkey []byte) {
n := len(bkey)
if n < 56 {
panic(fmt.Sprintf("bkey too short min size is 56 but we see %v in '%v'", n, string(bkey)))
}
beforeCkey := bkey[n-26 : n-20]
if !bytes.Equal(beforeCkey, ckeyPartExpected) {
panic(fmt.Sprintf(`bkey did not have expected ";ckey@" at 26 bytes from the end of the bkey '%v'; instead had '%v'`, string(bkey), string(beforeCkey)))
}
}
func bitmapAsString(rbm *roaring.Bitmap) (r string) {
r = "c("
slc := rbm.Slice()
@ -1380,30 +1388,7 @@ func bitmapAsString(rbm *roaring.Bitmap) (r string) {
return r + ")"
}
// should really be exported from the pilosa/roaring package so we don't get out of sync...
const (
containerNil byte = iota // no container
containerArray // slice of bit position values
containerBitmap // slice of 1024 uint64s
containerRun // container of run-encoded bits
)
var ckeyPartExpected = []byte(";ckey@")
func invName(rbfName string) (field, view string, shard uint64) {
s := strings.Split(rbfName, "\x00")
if len(s) != 3 {
panic("should have 3 parts")
}
field = s[0]
view = s[1]
var err error
shard, err = strconv.ParseUint(s[2], 10, 64)
panicOn(err)
return
}
func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName, index string) (s string) {
func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string) (s string) {
by := containerToBytes(ct)
hash := blake3sum16(by)
@ -1413,8 +1398,7 @@ func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName, index string) (s
rbm := &roaring.Bitmap{Containers: cts}
srbm := bitmapAsString(rbm)
field, view, shard := invName(rrName)
bkey := string(badgerKey(index, field, view, shard, ckey))
bkey := rrName + fmt.Sprintf("%020d", ckey)
s = fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, ct.N())
s += " ......." + srbm + "\n"
@ -1422,7 +1406,6 @@ func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName, index string) (s
}
func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
// begin write boilerplate
if tx.db == nil {
err = ErrTxClosed
@ -1484,6 +1467,7 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear
}
if clear {
existN := oldC.N() // number of bits set in the old container
newC := oldC.Difference(synthC)
@ -1496,14 +1480,6 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear
changes := int(existN - newC.N())
changed += changes
rowSet[currRow] -= changes
if tx.DeleteEmptyContainer && newC.N() == 0 {
err = tx.RemoveContainer(name, itrKey)
if err != nil {
return
}
continue
}
err = tx.PutContainer(name, itrKey, newC)
if err != nil {
return
@ -1529,9 +1505,9 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear
continue
}
newC := oldC.UnionInPlace(synthC)
newC := roaring.Union(oldC, synthC) // UnionInPlace was giving us crashes on overly large containers.
if roaring.ContainerType(newC) == containerBitmap {
if roaring.ContainerType(newC) == roaring.ContainerBitmap {
newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it.
}
if newC.N() != existN {

10
rbf/tx_test.go
View file

@ -21,6 +21,7 @@ import (
"time"
"github.com/pilosa/pilosa/v2/rbf"
"github.com/pilosa/pilosa/v2/txpath"
)
func TestTx_CommitRollback(t *testing.T) {
@ -472,7 +473,8 @@ func TestTx_Dump(t *testing.T) {
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(15)
nm := rbfName(field, view, shard)
nm := rbfName(index, field, view, shard)
if err := tx.CreateBitmap(nm); err != nil {
t.Fatal(err)
@ -481,12 +483,12 @@ func TestTx_Dump(t *testing.T) {
}
// test that we don't crash, and get *something* back
s := tx.DumpString(index)
s := tx.DumpString()
if s == "" {
panic("should have had 3 containers!")
}
}
func rbfName(field, view string, shard uint64) string {
return fmt.Sprintf("%s\x00%s\x00%d", field, view, shard)
func rbfName(index, field, view string, shard uint64) string {
return string(txpath.Prefix(index, field, view, shard))
}

39
rbf/vprint.go
View file

@ -29,8 +29,10 @@ import (
"io"
"os"
"path"
"path/filepath"
"runtime"
"runtime/debug"
"strings"
"sync"
"time"
)
@ -167,3 +169,40 @@ func Caller(upStack int) string {
}
var _ = stack // happy linter
var _ = listFilesUnderDir
// listFilesUnderDir returns the paths of files found under directory root.
// If includeRoot is true, it returns the full path, otherwise paths are relative to root.
// If requriedSuffix is supplied, the returned file paths will end in that,
// and any other files found during the walk of the directory tree will be ignored.
// If ignoreEmpty is true, files of size 0 will be excluded.
func listFilesUnderDir(root string, includeRoot bool, requiredSuffix string, ignoreEmpty bool) (files []string, err error) {
if !DirExists(root) {
return nil, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root)
}
n := len(root) + 1
if includeRoot {
n = 0
}
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if len(path) < n {
// ignore
} else {
if info == nil {
panic(fmt.Sprintf("info was nil for path = '%v'", path))
}
if info.IsDir() {
// skip directories.
} else {
if ignoreEmpty && info.Size() == 0 {
return nil
}
if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) {
files = append(files, path[n:])
}
}
}
return nil
})
return
}

64
roaring/container_stash.go
View file

@ -76,12 +76,12 @@ func (c *Container) String() string {
froze = c.flags.String()
}
switch c.typeID {
case containerArray:
case ContainerArray:
return fmt.Sprintf("<%s%sarray container, N=%d>", froze, space, c.N())
case containerBitmap:
case ContainerBitmap:
return fmt.Sprintf("<%s%sbitmap container, N=%d>",
froze, space, c.N())
case containerRun:
case ContainerRun:
return fmt.Sprintf("<%s%srun container, N=%d, len %dx interval>",
froze, space, c.N(), len(c.runs()))
default:
@ -105,7 +105,7 @@ func NewContainerBitmap(n int, bitmap []uint64) *Container {
if bitmap == nil {
return NewContainerBitmapN(nil, 0)
}
c := &Container{typeID: containerBitmap}
c := &Container{typeID: ContainerBitmap}
if len(bitmap) != bitmapN {
// adjust to required length
c.setBitmapCopy(bitmap)
@ -128,7 +128,7 @@ func NewContainerBitmapN(bitmap []uint64, n int32) *Container {
if bitmap == nil {
bitmap = make([]uint64, bitmapN)
}
c := &Container{typeID: containerBitmap, n: n}
c := &Container{typeID: ContainerBitmap, n: n}
if len(bitmap) != bitmapN {
// adjust to required length
c.setBitmapCopy(bitmap)
@ -141,7 +141,7 @@ func NewContainerBitmapN(bitmap []uint64, n int32) *Container {
// NewContainerArray returns an array container using the provided set of
// values. It's okay if the slice is nil; that's a length of zero.
func NewContainerArray(set []uint16) *Container {
c := &Container{typeID: containerArray}
c := &Container{typeID: ContainerArray}
c.setArray(set)
return c
}
@ -150,7 +150,7 @@ func NewContainerArray(set []uint16) *Container {
// values. It's okay if the slice is nil; that's a length of zero. It copies
// the provided slice to new storage.
func NewContainerArrayCopy(set []uint16) *Container {
c := &Container{typeID: containerArray}
c := &Container{typeID: ContainerArray}
c.setArrayMaybeCopy(set, true)
return c
}
@ -166,7 +166,7 @@ func NewContainerArrayN(set []uint16, n int32) *Container {
// NewContainerRun creates a new run container using a provided (possibly nil)
// slice of intervals.
func NewContainerRun(set []Interval16) *Container {
c := &Container{typeID: containerRun}
c := &Container{typeID: ContainerRun}
c.setRuns(set)
for _, run := range set {
c.n += int32(run.Last-run.Start) + 1
@ -177,7 +177,7 @@ func NewContainerRun(set []Interval16) *Container {
// NewContainerRunCopy creates a new run container using a provided (possibly nil)
// slice of intervals. It copies the provided slice to new storage.
func NewContainerRunCopy(set []Interval16) *Container {
c := &Container{typeID: containerRun}
c := &Container{typeID: ContainerRun}
c.setRunsMaybeCopy(set, true)
for _, run := range set {
c.n += int32(run.Last-run.Start) + 1
@ -188,7 +188,7 @@ func NewContainerRunCopy(set []Interval16) *Container {
// NewContainerRunN creates a new run array using a provided (possibly nil)
// slice of intervals. It overrides n using the provided value.
func NewContainerRunN(set []Interval16, n int32) *Container {
c := &Container{typeID: containerRun, n: n}
c := &Container{typeID: ContainerRun, n: n}
c.setRuns(set)
return c
}
@ -232,7 +232,7 @@ func (c *Container) setN(n int32) {
func (c *Container) typ() byte {
if c == nil {
return containerNil
return ContainerNil
}
return c.typeID
}
@ -299,11 +299,11 @@ func (c *Container) unmapOrClone() *Container {
c.flags &^= flagPristine
// mapped: we want to unmap the storage.
switch c.typeID {
case containerArray:
case ContainerArray:
c.setArrayMaybeCopy(c.array(), true)
case containerRun:
case ContainerRun:
c.setRunsMaybeCopy(c.runs(), true)
case containerBitmap:
case ContainerBitmap:
c.setBitmapCopy(c.bitmap())
default:
panic(fmt.Sprintf("can't thaw invalid container, type %d", c.typeID))
@ -317,7 +317,7 @@ func (c *Container) array() []uint16 {
panic("attempt to read a nil container's array")
}
if roaringParanoia {
if c.typeID != containerArray {
if c.typeID != ContainerArray {
panic("attempt to read non-array's array")
}
}
@ -332,7 +332,7 @@ func (c *Container) setArrayMaybeCopy(array []uint16, doCopy bool) {
if c == nil || c.frozen() {
panic("setArray on nil or frozen container")
}
if c.typeID != containerArray {
if c.typeID != ContainerArray {
panic("attempt to write non-array's array")
}
}
@ -376,7 +376,7 @@ func (c *Container) bitmap() []uint64 {
panic("attempt to read nil container's bitmap")
}
if roaringParanoia {
if c.typeID != containerBitmap {
if c.typeID != ContainerBitmap {
panic("attempt to read non-bitmap's bitmap")
}
}
@ -387,7 +387,7 @@ func (c *Container) bitmap() []uint64 {
// is provided. The target should be zeroed, or this becomes an implicit
// union.
func (c *Container) AsBitmap(target []uint64) (out []uint64) {
if c != nil && c.typeID == containerBitmap {
if c != nil && c.typeID == ContainerBitmap {
return c.bitmap()
}
// Reminder: len(nil) == 0.
@ -403,14 +403,14 @@ func (c *Container) AsBitmap(target []uint64) (out []uint64) {
if c == nil {
return out
}
if c.typeID == containerArray {
if c.typeID == ContainerArray {
a := c.array()
for _, v := range a {
out[v/64] |= 1 << (v % 64)
}
return out
}
if c.typeID == containerRun {
if c.typeID == ContainerRun {
runs := c.runs()
b := (*[1024]uint64)(unsafe.Pointer(&out[0]))
for _, r := range runs {
@ -478,7 +478,7 @@ func (c *Container) setBitmap(bitmap []uint64) {
panic("setBitmap on nil or frozen container")
}
if roaringParanoia {
if c.typeID != containerBitmap {
if c.typeID != ContainerBitmap {
panic("attempt to write non-bitmap's bitmap")
}
}
@ -495,7 +495,7 @@ func (c *Container) runs() []Interval16 {
panic("attempt to read nil container's runs")
}
if roaringParanoia {
if c.typeID != containerRun {
if c.typeID != ContainerRun {
panic("attempt to read non-run's runs")
}
}
@ -514,7 +514,7 @@ func (c *Container) setRunsMaybeCopy(runs []Interval16, doCopy bool) {
if c == nil || c.frozen() {
panic("setRuns on nil or frozen container")
}
if c.typeID != containerRun {
if c.typeID != ContainerRun {
panic("attempt to write non-run's runs")
}
}
@ -548,9 +548,9 @@ func (c *Container) setRunsMaybeCopy(runs []Interval16, doCopy bool) {
func (c *Container) UpdateOrMake(typ byte, n int32, mapped bool) *Container {
if c == nil {
switch typ {
case containerRun:
case ContainerRun:
c = NewContainerRunN(nil, n)
case containerBitmap:
case ContainerBitmap:
c = NewContainerBitmapN(nil, n)
default:
c = NewContainerArrayN(nil, n)
@ -567,9 +567,9 @@ func (c *Container) UpdateOrMake(typ byte, n int32, mapped bool) *Container {
c.setMapped(mapped)
// we don't know that any existing slice is usable, so let's ditch it
switch c.typeID {
case containerArray:
case ContainerArray:
c.pointer, c.len, c.cap = &c.data[0], 0, stashedArraySize
case containerRun:
case ContainerRun:
c.pointer, c.len, c.cap = &c.data[0], 0, stashedRunSize
default:
c.pointer, c.len, c.cap = nil, 0, 0
@ -590,9 +590,9 @@ func (c *Container) Update(typ byte, n int32, mapped bool) {
c.setMapped(mapped)
// we don't know that any existing slice is usable, so let's ditch it
switch c.typeID {
case containerArray:
case ContainerArray:
c.pointer, c.len, c.cap = nil, 0, 0
case containerRun:
case ContainerRun:
c.pointer, c.len, c.cap = nil, 0, 0
default:
c.pointer, c.len, c.cap = nil, 0, 0
@ -604,7 +604,7 @@ func (c *Container) isArray() bool {
if c == nil {
panic("calling isArray on nil container")
}
return c.typeID == containerArray
return c.typeID == ContainerArray
}
// isBitmap returns true if the container is a bitmap container.
@ -612,7 +612,7 @@ func (c *Container) isBitmap() bool {
if c == nil {
panic("calling isBitmap on nil container")
}
return c.typeID == containerBitmap
return c.typeID == ContainerBitmap
}
// isRun returns true if the container is a run-length-encoded container.
@ -620,5 +620,5 @@ func (c *Container) isRun() bool {
if c == nil {
panic("calling isRun on nil container")
}
return c.typeID == containerRun
return c.typeID == ContainerRun
}

6
roaring/containers_test.go
View file

@ -119,8 +119,10 @@ func TestSliceContainers(t *testing.T) {
if c == nil {
t.Fatalf("Get(%d) returned nil container", key)
}
if c.data[0] != set[0] {
t.Fatalf("Get(%d): expected: %v, got: %v", key, set[0], c.data[0])
if len(c.data) > 0 { // happy linter
if c.data[0] != set[0] {
t.Fatalf("Get(%d): expected: %v, got: %v", key, set[0], c.data[0])
}
}
}
})

184
roaring/roaring.go
View file

@ -62,17 +62,17 @@ const (
)
const (
containerNil byte = iota // no container
containerArray // slice of bit position values
containerBitmap // slice of 1024 uint64s
containerRun // container of run-encoded bits
ContainerNil byte = iota // no container
ContainerArray // slice of bit position values
ContainerBitmap // slice of 1024 uint64s
ContainerRun // container of run-encoded bits
)
// map used for a more descriptive print
var containerTypeNames = map[byte]string{
containerArray: "array",
containerBitmap: "bitmap",
containerRun: "run",
ContainerArray: "array",
ContainerBitmap: "bitmap",
ContainerRun: "run",
}
var fullContainer = NewContainerRun([]Interval16{{Start: 0, Last: MaxContainerVal}}).Freeze()
@ -845,33 +845,33 @@ func (c *Container) intersectInPlace(other *Container) *Container {
}
switch c.typ() {
case containerArray:
case ContainerArray:
switch other.typ() {
case containerArray:
case ContainerArray:
return intersectArrayArrayInPlace(c, other)
case containerBitmap:
case ContainerBitmap:
return intersectArrayBitmapInPlace(c, other)
case containerRun:
case ContainerRun:
return intersectArrayRunInPlace(c, other)
}
case containerBitmap:
case ContainerBitmap:
switch other.typ() {
case containerArray:
case ContainerArray:
return intersectBitmapArrayInPlace(c, other)
case containerBitmap:
case ContainerBitmap:
return intersectBitmapBitmapInPlace(c, other)
case containerRun:
case ContainerRun:
return intersectBitmapRunInPlace(c, other)
}
case containerRun:
case ContainerRun:
switch other.typ() {
case containerArray:
case ContainerArray:
return intersectRunArrayInPlace(c, other)
case containerBitmap:
case ContainerBitmap:
return intersectRunBitmapInPlace(c, other)
case containerRun:
case ContainerRun:
return intersectRunRunInPlace(c, other)
}
}
@ -881,17 +881,17 @@ func (c *Container) intersectInPlace(other *Container) *Container {
func (c *Container) copyInPlace(other *Container) *Container {
switch other.typ() {
case containerArray:
c.setTyp(containerArray)
case ContainerArray:
c.setTyp(ContainerArray)
c.setArrayMaybeCopy(other.array(), true)
case containerBitmap:
c.setTyp(containerBitmap)
case ContainerBitmap:
c.setTyp(ContainerBitmap)
c.setBitmapCopy(other.bitmap())
c.setN(other.N())
case containerRun:
c.setTyp(containerRun)
case ContainerRun:
c.setTyp(ContainerRun)
c.setRunsMaybeCopy(other.runs(), true)
c.setN(other.N())
@ -1026,7 +1026,7 @@ func intersectBitmapArrayInPlace(a, b *Container) *Container {
}
}
array = array[:n]
a.setTyp(containerArray)
a.setTyp(ContainerArray)
a.setArray(array)
return a
@ -1154,7 +1154,7 @@ func intersectRunArrayInPlace(a, b *Container) *Container {
}
array = array[:n]
a.setTyp(containerArray)
a.setTyp(ContainerArray)
a.setArray(array)
return a
@ -1407,7 +1407,7 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
// first other container, but for some cases, that will
// result in cloning a non-bitmap, then converting it
// to a bitmap, and this will be expensive...
if expectedN >= 512 && iContainer.typ() != containerBitmap {
if expectedN >= 512 && iContainer.typ() != ContainerBitmap {
// copying the non-bitmap, then converting it,
// is expensive.
statsHit("unionInPlace/newBitmap")
@ -1428,12 +1428,12 @@ func (b *Bitmap) unionInPlace(others ...*Bitmap) {
// convert it preemptively, because union into a
// bitmap is nearly always faster.
itersToUnion = bitmapIters[i:]
if expectedN >= 512 && tContainer.typ() != containerBitmap {
if expectedN >= 512 && tContainer.typ() != ContainerBitmap {
statsHit("unionInPlace/convertToBitmap")
switch tContainer.typ() {
case containerArray:
case ContainerArray:
tContainer = tContainer.arrayToBitmap()
case containerRun:
case ContainerRun:
tContainer = tContainer.runToBitmap()
}
}
@ -1967,7 +1967,7 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in
// a run container keeps its data after an initial 2 byte length header
var runCount uint16
if r.currentType == containerRun {
if r.currentType == ContainerRun {
runCount = binary.LittleEndian.Uint16(r.data[r.currentDataOffset : r.currentDataOffset+runCountHeaderSize])
r.currentDataOffset += 2
}
@ -1979,13 +1979,13 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in
r.currentPointer = (*uint16)(unsafe.Pointer(&r.data[r.currentDataOffset]))
var size int
switch r.currentType {
case containerArray:
case ContainerArray:
r.currentLen = r.currentN
size = r.currentLen * 2
case containerBitmap:
case ContainerBitmap:
r.currentLen = 1024
size = 8192
case containerRun:
case ContainerRun:
r.currentLen = int(runCount)
size = r.currentLen * 4
}
@ -2035,7 +2035,7 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length
}
// a run container keeps its data after an initial 2 byte length header
var runCount uint16
if r.currentType == containerRun {
if r.currentType == ContainerRun {
if int(r.currentDataOffset)+2 > len(r.data) {
r.Done(fmt.Errorf("insufficient data for offsets container %d/%d, expect run length at %d/%d bytes",
r.currentIdx, r.keys, r.currentDataOffset, len(r.data)))
@ -2052,13 +2052,13 @@ func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length
r.currentPointer = (*uint16)(unsafe.Pointer(&r.data[r.currentDataOffset]))
var size int
switch r.currentType {
case containerArray:
case ContainerArray:
r.currentLen = r.currentN
size = r.currentLen * 2
case containerBitmap:
case ContainerBitmap:
r.currentLen = 1024
size = 8192
case containerRun:
case ContainerRun:
// official format stores runs as start/len, we want to convert, but since
// they might be mmapped, we can't write to that memory
newRuns := make([]Interval16, runCount)
@ -2257,7 +2257,7 @@ func (b *Bitmap) ImportRoaringRawIterator(itr RoaringIterator, clear bool, log b
return newerC, true
}
newC = oldC.unionInPlace(&synthC)
if newC.typeID == containerBitmap {
if newC.typeID == ContainerBitmap {
newC.Repair()
}
if newC.N() != existN {
@ -2461,12 +2461,12 @@ func BitmapsToRoaring(bitmaps []*Bitmap) []byte {
binary.LittleEndian.PutUint32(offset[0:4], uint32(dataOffset+int(offsetEnd)))
nextData := data[dataOffset:]
switch c.typeID { // TODO: make this work on big endian machines
case containerArray:
case ContainerArray:
dataOffset += 2 * copy((*[1 << 16]uint16)(unsafe.Pointer(&nextData[0]))[:], c.array())
case containerBitmap:
case ContainerBitmap:
copy((*[1024]uint64)(unsafe.Pointer(&nextData[0]))[:], c.bitmap())
dataOffset += 8192
case containerRun:
case ContainerRun:
binary.LittleEndian.PutUint16(nextData[0:2], uint16(c.len))
dataOffset += 2
dataOffset += 4 * copy((*[1 << 15]Interval16)(unsafe.Pointer(&nextData[2]))[:], c.runs())
@ -2489,11 +2489,11 @@ func (b *Bitmap) roaringSize() (int64, int64) {
}
count++
switch c.typeID {
case containerArray:
case ContainerArray:
size += 2 * int64(c.N())
case containerBitmap:
case ContainerBitmap:
size += 8192
case containerRun:
case ContainerRun:
// 2 bytes for the count of runs, plus 4 bytes per run
size += 2 + (4 * int64(c.len))
}
@ -3141,39 +3141,39 @@ func (c *Container) optimize() *Container {
var newType byte
if runs <= runMaxSize && runs <= c.N()/2 {
newType = containerRun
newType = ContainerRun
} else if c.N() < ArrayMaxSize {
newType = containerArray
newType = ContainerArray
} else {
newType = containerBitmap
newType = ContainerBitmap
}
// Then convert accordingly.
if c.isArray() {
if newType == containerBitmap {
if newType == ContainerBitmap {
statsHit("optimize/arrayToBitmap")
c = c.arrayToBitmap()
} else if newType == containerRun {
} else if newType == ContainerRun {
statsHit("optimize/arrayToRun")
c = c.arrayToRun(runs)
} else {
statsHit("optimize/arrayUnchanged")
}
} else if c.isBitmap() {
if newType == containerArray {
if newType == ContainerArray {
statsHit("optimize/bitmapToArray")
c = c.bitmapToArray()
} else if newType == containerRun {
} else if newType == ContainerRun {
statsHit("optimize/bitmapToRun")
c = c.bitmapToRun(runs)
} else {
statsHit("optimize/bitmapUnchanged")
}
} else if c.isRun() {
if newType == containerBitmap {
if newType == ContainerBitmap {
statsHit("optimize/runToBitmap")
c = c.runToBitmap()
} else if newType == containerArray {
} else if newType == ContainerArray {
statsHit("optimize/runToArray")
c = c.runToArray()
} else {
@ -3202,36 +3202,36 @@ func (c *Container) unionInPlace(other *Container) *Container {
return fullContainer
}
switch c.typ() {
case containerBitmap:
case ContainerBitmap:
switch other.typ() {
case containerBitmap:
case ContainerBitmap:
return unionBitmapBitmapInPlace(c, other)
case containerArray:
case ContainerArray:
return unionBitmapArrayInPlace(c, other)
case containerRun:
case ContainerRun:
return unionBitmapRunInPlace(c, other)
}
case containerArray:
case ContainerArray:
switch other.typ() {
case containerBitmap:
case ContainerBitmap:
c = c.arrayToBitmap()
return unionBitmapBitmapInPlace(c, other)
case containerArray:
case ContainerArray:
return unionArrayArrayInPlace(c, other)
case containerRun:
case ContainerRun:
c = c.arrayToBitmap()
return unionBitmapRunInPlace(c, other)
}
case containerRun:
case ContainerRun:
switch other.typ() {
case containerBitmap:
case ContainerBitmap:
c = c.runToBitmap()
return unionBitmapBitmapInPlace(c, other)
case containerArray:
case ContainerArray:
c = c.runToBitmap()
return unionBitmapArrayInPlace(c, other)
case containerRun:
case ContainerRun:
return unionRunRunInPlace(c, other)
}
}
@ -3412,7 +3412,7 @@ func (c *Container) bitmapToArray() *Container {
if c.frozen() {
return NewContainerArray(nil)
}
c.setTyp(containerArray)
c.setTyp(ContainerArray)
c.setArray(nil)
return c
}
@ -3441,7 +3441,7 @@ func (c *Container) bitmapToArray() *Container {
if c.frozen() {
return NewContainerArray(array)
}
c.setTyp(containerArray)
c.setTyp(ContainerArray)
c.setMapped(false)
c.setArray(array)
return c
@ -3462,7 +3462,7 @@ func (c *Container) arrayToBitmap() *Container {
if c.frozen() {
return NewContainerBitmap(0, nil)
}
c.setTyp(containerBitmap)
c.setTyp(ContainerBitmap)
c.setBitmap(make([]uint64, bitmapN))
return c
}
@ -3474,7 +3474,7 @@ func (c *Container) arrayToBitmap() *Container {
if c.frozen() {
return NewContainerBitmapN(bitmap, c.N())
}
c.setTyp(containerBitmap)
c.setTyp(ContainerBitmap)
c.setMapped(false)
c.setBitmap(bitmap)
return c
@ -3495,7 +3495,7 @@ func (c *Container) runToBitmap() *Container {
if c.frozen() {
return NewContainerBitmap(0, nil)
}
c.setTyp(containerBitmap)
c.setTyp(ContainerBitmap)
c.setBitmap(make([]uint64, bitmapN))
return c
}
@ -3533,7 +3533,7 @@ func (c *Container) runToBitmap() *Container {
if c.frozen() {
return NewContainerBitmapN(bitmap, c.N())
}
c.setTyp(containerBitmap)
c.setTyp(ContainerBitmap)
c.setMapped(false)
c.setBitmap(bitmap)
return c
@ -3554,7 +3554,7 @@ func (c *Container) bitmapToRun(numRuns int32) *Container {
if c.frozen() {
return NewContainerRun(nil)
}
c.setTyp(containerRun)
c.setTyp(ContainerRun)
c.setRuns(nil)
return c
}
@ -3605,7 +3605,7 @@ func (c *Container) bitmapToRun(numRuns int32) *Container {
if c.frozen() {
return NewContainerRunN(runs, c.N())
}
c.setTyp(containerRun)
c.setTyp(ContainerRun)
c.setRuns(runs)
c.setMapped(false)
return c
@ -3626,7 +3626,7 @@ func (c *Container) arrayToRun(numRuns int32) *Container {
if c.frozen() {
return NewContainerRun(nil)
}
c.setTyp(containerRun)
c.setTyp(ContainerRun)
c.setRuns(nil)
return c
}
@ -3651,7 +3651,7 @@ func (c *Container) arrayToRun(numRuns int32) *Container {
if c.frozen() {
return NewContainerRunN(runs, c.N())
}
c.setTyp(containerRun)
c.setTyp(ContainerRun)
c.setMapped(false)
c.setRuns(runs)
return c
@ -3672,7 +3672,7 @@ func (c *Container) runToArray() *Container {
if c.frozen() {
return NewContainerArray(nil)
}
c.setTyp(containerArray)
c.setTyp(ContainerArray)
c.setArray(nil)
return c
}
@ -3695,7 +3695,7 @@ func (c *Container) runToArray() *Container {
if c.frozen() {
return NewContainerArray(array)
}
c.setTyp(containerArray)
c.setTyp(ContainerArray)
c.setMapped(false)
c.setArray(array)
return c
@ -3708,15 +3708,15 @@ func (c *Container) Clone() (out *Container) {
return nil
}
switch c.typ() {
case containerArray:
case ContainerArray:
statsHit("Container/Clone/Array")
out = NewContainerArrayCopy(c.array())
case containerBitmap:
case ContainerBitmap:
statsHit("Container/Clone/Bitmap")
other := NewContainerBitmapN(nil, c.N())
copy(other.bitmap(), c.bitmap())
out = other
case containerRun:
case ContainerRun:
statsHit("Container/Clone/Run")
out = NewContainerRunCopy(c.runs())
default:
@ -4717,15 +4717,15 @@ func (c *Container) BitwiseCompare(c2 *Container) error {
return nil
}
switch typePair(c.typ(), c2.typ()) {
case typePair(containerArray, containerArray):
case typePair(ContainerArray, ContainerArray):
return compareArrayArray(c.array(), c2.array())
case typePair(containerArray, containerBitmap):
case typePair(ContainerArray, ContainerBitmap):
return compareArrayBitmap(c.array(), c2.bitmap())
case typePair(containerBitmap, containerArray):
case typePair(ContainerBitmap, ContainerArray):
return compareArrayBitmap(c2.array(), c.bitmap())
case typePair(containerArray, containerRun):
case typePair(ContainerArray, ContainerRun):
return compareArrayRuns(c.array(), c2.runs())
case typePair(containerRun, containerArray):
case typePair(ContainerRun, ContainerArray):
return compareArrayRuns(c2.array(), c.runs())
default:
c3 := xor(c, c2)
@ -5432,7 +5432,7 @@ func xorArrayBitmap(a, b *Container) *Container {
// It's possible that output was converted from bitmap to array in output.remove()
// so we only do this conversion if output is still a bitmap container.
if output.typ() == containerBitmap && output.count() < ArrayMaxSize {
if output.typ() == ContainerBitmap && output.count() < ArrayMaxSize {
output = output.bitmapToArray()
}
@ -6239,9 +6239,9 @@ func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint
return size, containerTyper, header, pos, haveRuns, err
}
cf := func(index uint, card int) (newType byte) {
newType = containerBitmap
newType = ContainerBitmap
if card < ArrayMaxSize {
newType = containerArray
newType = ContainerArray
}
return newType
}
@ -6268,7 +6268,7 @@ func readOfficialHeader(buf []byte) (size uint32, containerTyper func(index uint
pos += isRunBitmapSize
containerTyper = func(index uint, card int) byte {
if isRunBitmap[index/8]&(1<<(index%8)) != 0 {
return containerRun
return ContainerRun
}
return cf(index, card)
}
@ -6714,7 +6714,7 @@ func differenceRunBitmapInPlace(c, other *Container) {
for i, word := range other.bitmap() {
bitmap[i] = ^word
}
c.setTyp(containerBitmap)
c.setTyp(ContainerBitmap)
c.setMapped(false)
c.setBitmap(bitmap)
c.setN(c.count())

72
roaring/roaring_helpers_test.go
View file

@ -252,12 +252,12 @@ type testOp struct {
func doContainer(typ byte, data interface{}) *Container {
switch typ {
case containerArray:
case ContainerArray:
return NewContainerArray(data.([]uint16))
case containerBitmap:
case ContainerBitmap:
c := NewContainerBitmap(-1, data.([]uint64))
return c
case containerRun:
case ContainerRun:
return NewContainerRun(data.([]Interval16))
}
return nil
@ -272,45 +272,45 @@ func setupContainerTests() map[byte]map[string]*Container {
sampleTestContainers = make(map[byte]map[string]*Container)
// array containers
sampleTestContainers[containerArray] = map[string]*Container{
"empty": doContainer(containerArray, arrayEmpty()),
"full": doContainer(containerArray, arrayFull()),
"firstBitSet": doContainer(containerArray, arrayFirstBitSet()),
"lastBitSet": doContainer(containerArray, arrayLastBitSet()),
"firstBitUnset": doContainer(containerArray, arrayFirstBitUnset()),
"lastBitUnset": doContainer(containerArray, arrayLastBitUnset()),
"innerBitsSet": doContainer(containerArray, arrayInnerBitsSet()),
"outerBitsSet": doContainer(containerArray, arrayOuterBitsSet()),
"oddBitsSet": doContainer(containerArray, arrayOddBitsSet()),
"evenBitsSet": doContainer(containerArray, arrayEvenBitsSet()),
sampleTestContainers[ContainerArray] = map[string]*Container{
"empty": doContainer(ContainerArray, arrayEmpty()),
"full": doContainer(ContainerArray, arrayFull()),
"firstBitSet": doContainer(ContainerArray, arrayFirstBitSet()),
"lastBitSet": doContainer(ContainerArray, arrayLastBitSet()),
"firstBitUnset": doContainer(ContainerArray, arrayFirstBitUnset()),
"lastBitUnset": doContainer(ContainerArray, arrayLastBitUnset()),
"innerBitsSet": doContainer(ContainerArray, arrayInnerBitsSet()),
"outerBitsSet": doContainer(ContainerArray, arrayOuterBitsSet()),
"oddBitsSet": doContainer(ContainerArray, arrayOddBitsSet()),
"evenBitsSet": doContainer(ContainerArray, arrayEvenBitsSet()),
}
// bitmap containers
sampleTestContainers[containerBitmap] = map[string]*Container{
"empty": doContainer(containerBitmap, bitmapEmpty()),
"full": doContainer(containerBitmap, bitmapFull()),
"firstBitSet": doContainer(containerBitmap, bitmapFirstBitSet()),
"lastBitSet": doContainer(containerBitmap, bitmapLastBitSet()),
"firstBitUnset": doContainer(containerBitmap, bitmapFirstBitUnset()),
"lastBitUnset": doContainer(containerBitmap, bitmapLastBitUnset()),
"innerBitsSet": doContainer(containerBitmap, bitmapInnerBitsSet()),
"outerBitsSet": doContainer(containerBitmap, bitmapOuterBitsSet()),
"oddBitsSet": doContainer(containerBitmap, bitmapOddBitsSet()),
"evenBitsSet": doContainer(containerBitmap, bitmapEvenBitsSet()),
sampleTestContainers[ContainerBitmap] = map[string]*Container{
"empty": doContainer(ContainerBitmap, bitmapEmpty()),
"full": doContainer(ContainerBitmap, bitmapFull()),
"firstBitSet": doContainer(ContainerBitmap, bitmapFirstBitSet()),
"lastBitSet": doContainer(ContainerBitmap, bitmapLastBitSet()),
"firstBitUnset": doContainer(ContainerBitmap, bitmapFirstBitUnset()),
"lastBitUnset": doContainer(ContainerBitmap, bitmapLastBitUnset()),
"innerBitsSet": doContainer(ContainerBitmap, bitmapInnerBitsSet()),
"outerBitsSet": doContainer(ContainerBitmap, bitmapOuterBitsSet()),
"oddBitsSet": doContainer(ContainerBitmap, bitmapOddBitsSet()),
"evenBitsSet": doContainer(ContainerBitmap, bitmapEvenBitsSet()),
}
// run containers
sampleTestContainers[containerRun] = map[string]*Container{
"empty": doContainer(containerRun, runEmpty()),
"full": doContainer(containerRun, runFull()),
"firstBitSet": doContainer(containerRun, runFirstBitSet()),
"lastBitSet": doContainer(containerRun, runLastBitSet()),
"firstBitUnset": doContainer(containerRun, runFirstBitUnset()),
"lastBitUnset": doContainer(containerRun, runLastBitUnset()),
"innerBitsSet": doContainer(containerRun, runInnerBitsSet()),
"outerBitsSet": doContainer(containerRun, runOuterBitsSet()),
"oddBitsSet": doContainer(containerRun, runOddBitsSet()),
"evenBitsSet": doContainer(containerRun, runEvenBitsSet()),
sampleTestContainers[ContainerRun] = map[string]*Container{
"empty": doContainer(ContainerRun, runEmpty()),
"full": doContainer(ContainerRun, runFull()),
"firstBitSet": doContainer(ContainerRun, runFirstBitSet()),
"lastBitSet": doContainer(ContainerRun, runLastBitSet()),
"firstBitUnset": doContainer(ContainerRun, runFirstBitUnset()),
"lastBitUnset": doContainer(ContainerRun, runLastBitUnset()),
"innerBitsSet": doContainer(ContainerRun, runInnerBitsSet()),
"outerBitsSet": doContainer(ContainerRun, runOuterBitsSet()),
"oddBitsSet": doContainer(ContainerRun, runOddBitsSet()),
"evenBitsSet": doContainer(ContainerRun, runEvenBitsSet()),
}
})

18
roaring/roaring_internal_test.go
View file

@ -2962,7 +2962,7 @@ func TestContainerCombinations(t *testing.T) {
cts := setupContainerTests()
containerTypes := []byte{containerArray, containerBitmap, containerRun}
containerTypes := []byte{ContainerArray, ContainerBitmap, ContainerRun}
testOps := []testOp{
// intersect
@ -4323,8 +4323,8 @@ func TestBitmapAny(t *testing.T) {
}
func TestDifferenceInPlace_N(t *testing.T) {
a := doContainer(containerRun, runFull())
b := doContainer(containerBitmap, bitmapFull())
a := doContainer(ContainerRun, runFull())
b := doContainer(ContainerBitmap, bitmapFull())
r := differenceInPlaceWrapper(a, b)
if r.N() != 0 {
t.Error("expected difference of containers to have n=0")
@ -4352,8 +4352,8 @@ func BenchmarkUnionRunRunInPlace(bm *testing.B) {
for _, br := range runs {
bm.Run("RunToBitmapRun-"+ar.name+"_"+br.name, func(bm *testing.B) {
for i := 0; i < bm.N; i++ {
arun := doContainer(containerRun, ar.fn())
brun := doContainer(containerRun, br.fn())
arun := doContainer(ContainerRun, ar.fn())
brun := doContainer(ContainerRun, br.fn())
abmp := arun.runToBitmap()
unionBitmapRunInPlace(abmp, brun)
@ -4362,8 +4362,8 @@ func BenchmarkUnionRunRunInPlace(bm *testing.B) {
bm.Run("RunRun-"+ar.name+"_"+br.name, func(bm *testing.B) {
for i := 0; i < bm.N; i++ {
arun := doContainer(containerRun, ar.fn())
brun := doContainer(containerRun, br.fn())
arun := doContainer(ContainerRun, ar.fn())
brun := doContainer(ContainerRun, br.fn())
unionRunRunInPlace(arun, brun)
}
@ -4390,8 +4390,8 @@ func TestUnionRunRunInPlaceBitwiseCompare(t *testing.T) {
for _, a := range runs {
for _, b := range runs {
t.Run(a.name+"-"+b.name, func(t *testing.T) {
arun := doContainer(containerRun, a.run)
brun := doContainer(containerRun, b.run)
arun := doContainer(ContainerRun, a.run)
brun := doContainer(ContainerRun, b.run)
out1 := unionBitmapRunInPlace(arun.runToBitmap(), brun)
out2 := unionRunRunInPlace(arun, brun)

18
roaring/unmarshal_binary.go
View file

@ -49,11 +49,11 @@ func (b *Bitmap) UnmarshalBinary(data []byte) (err error) {
for itrErr == nil {
var newC *Container
switch itrCType {
case containerArray:
case ContainerArray:
newC = NewContainerArray((*[4096]uint16)(unsafe.Pointer(itrPointer))[:itrLen:itrLen])
case containerRun:
case ContainerRun:
newC = NewContainerRunN((*[2048]Interval16)(unsafe.Pointer(itrPointer))[:itrLen:itrLen], int32(itrN))
case containerBitmap:
case ContainerBitmap:
newC = NewContainerBitmapN((*[1024]uint64)(unsafe.Pointer(itrPointer))[:1024:itrLen], int32(itrN))
default:
panic("invalid container type")
@ -132,20 +132,20 @@ func InspectBinary(data []byte, mapped bool, info *BitmapInfo) (b *Bitmap, mappe
for itrErr == nil {
var size int
switch itrCType {
case containerArray:
case ContainerArray:
size = int(itrN) * 2
case containerBitmap:
case ContainerBitmap:
size = 8192
case containerRun:
case ContainerRun:
size = itrLen*interval16Size + runCountHeaderSize
}
var newC *Container
switch itrCType {
case containerArray:
case ContainerArray:
newC = NewContainerArray((*[4096]uint16)(unsafe.Pointer(itrPointer))[:itrLen:itrLen])
case containerRun:
case ContainerRun:
newC = NewContainerRunN((*[2048]Interval16)(unsafe.Pointer(itrPointer))[:itrLen:itrLen], int32(itrN))
case containerBitmap:
case ContainerBitmap:
newC = NewContainerBitmapN((*[1024]uint64)(unsafe.Pointer(itrPointer))[:1024:itrLen], int32(itrN))
default:
panic("invalid container type")

695
rrtx.go Normal file
View file

@ -0,0 +1,695 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"sync"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
// MultiTx implements the transaction interface to combine multiple transactions.
type MultiTx struct {
mu sync.Mutex
writable bool
holder *Holder
index *Index
txs map[multiTxKey]Tx
}
// NewMultiTx returns a new instance of MultiTx for a Holder.
func NewMultiTx(writable bool, holder *Holder) *MultiTx {
return &MultiTx{
writable: writable,
holder: holder,
txs: make(map[multiTxKey]Tx),
}
}
// NewMultiTxWithIndex returns a new instance of MultiTx for a single index.
func NewMultiTxWithIndex(writable bool, index *Index) *MultiTx {
return &MultiTx{
writable: writable,
index: index,
txs: make(map[multiTxKey]Tx),
}
}
var _ Tx = (*MultiTx)(nil)
func (mtx *MultiTx) Type() string {
return RoaringTxn
}
// debugging, what does this Tx see as its database?
func (mtx *MultiTx) Dump() {
mtx.mu.Lock()
defer mtx.mu.Unlock()
if len(mtx.txs) == 0 {
return
}
for _, tx := range mtx.txs {
tx.Dump()
return
}
}
func (mtx *MultiTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
tx, err := mtx.txNoShard(index)
panicOn(err)
return tx.SliceOfShards(index, field, view, optionalViewPath)
}
func (mtx *MultiTx) UseRowCache() bool {
return true
}
func (mtx *MultiTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
}
func (mtx *MultiTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.NewTxIterator(index, field, view, shard)
}
// Readonly is true if the transaction is not read-and-write, but only doing reads.
func (mtx *MultiTx) Readonly() bool {
return !mtx.writable
}
func (mtx *MultiTx) Pointer() string {
return fmt.Sprintf("%p", mtx)
}
func (mtx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
}
func (mtx *MultiTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
tx, err := mtx.tx(index, shard)
panicOn(err)
tx.IncrementOpN(index, field, view, shard, changedN)
}
// Rollback rolls back all underlying transactions.
func (mtx *MultiTx) Rollback() {
for _, tx := range mtx.txs {
tx.Rollback()
}
}
// Commit commits all underlying transactions.
func (mtx *MultiTx) Commit() (err error) {
for _, tx := range mtx.txs {
if e := tx.Commit(); e != nil && err == nil {
err = e
}
}
return err
}
func (mtx *MultiTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.RoaringBitmap(index, field, view, shard)
}
func (mtx *MultiTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.Container(index, field, view, shard, key)
}
func (mtx *MultiTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.PutContainer(index, field, view, shard, key, c)
}
func (mtx *MultiTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.RemoveContainer(index, field, view, shard, key)
}
func (mtx *MultiTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Add(index, field, view, shard, batched, a...)
}
func (mtx *MultiTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Remove(index, field, view, shard, a...)
}
func (mtx *MultiTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return false, err
}
return tx.Contains(index, field, view, shard, v)
}
func (mtx *MultiTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, false, err
}
return tx.ContainerIterator(index, field, view, shard, key)
}
func (mtx *MultiTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.ForEach(index, field, view, shard, fn)
}
func (mtx *MultiTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.ForEachRange(index, field, view, shard, start, end, fn)
}
func (mtx *MultiTx) Count(index, field, view string, shard uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Count(index, field, view, shard)
}
func (mtx *MultiTx) Max(index, field, view string, shard uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Max(index, field, view, shard)
}
func (mtx *MultiTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, false, err
}
return tx.Min(index, field, view, shard)
}
func (mtx *MultiTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.UnionInPlace(index, field, view, shard, others...)
}
func (mtx *MultiTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.CountRange(index, field, view, shard, start, end)
}
func (mtx *MultiTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.OffsetRange(index, field, view, shard, offset, start, end)
}
// tx returns a transaction by index/shard. Reuses transaction if already open.
// Otherwise begins a new transaction.
func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
mkey := multiTxKey{index: index, shard: shard, write: mtx.writable}
// Lookup transaction from cache.
tx := mtx.txs[mkey]
if tx != nil {
return tx, nil
}
// If transaction doesn't exist, lookup the index.
idx := mtx.index
if mtx.holder != nil {
if idx = mtx.holder.Index(index); idx == nil {
return nil, ErrIndexNotFound
}
}
// Begin tranaction & cache it.
if tx, err = idx.BeginTx(mtx.writable, shard); err != nil {
return nil, err
}
mtx.txs[mkey] = tx
return tx, nil
}
// version of the above for SliceOfShards(), where we don't have a shard.
func (mtx *MultiTx) txNoShard(index string) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
// Lookup transaction from cache.
for _, tx := range mtx.txs {
if tx.(*RoaringTx).Index.name == index {
return tx, nil
}
}
panic(fmt.Sprintf("no prior RoaringTx available, looking up index='%v'", index))
}
type multiTxKey struct {
index string
shard uint64
write bool
}
// RoaringTx represents a fake transaction object for Roaring storage.
type RoaringTx struct {
write bool
Index *Index
Field *Field
fragment *fragment
}
func (tx *RoaringTx) Type() string {
return RoaringTxn
}
func (tx *RoaringTx) Dump() {
fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys())
}
func (tx *RoaringTx) UseRowCache() bool {
return true
}
func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
// SliceOfShards is based on view.openFragments()
file, err := os.Open(filepath.Join(optionalViewPath, "fragments"))
if os.IsNotExist(err) {
return
} else if err != nil {
return nil, errors.Wrap(err, "opening fragments directory")
}
defer file.Close()
fis, err := file.Readdir(0)
if err != nil {
return nil, errors.Wrap(err, "reading fragments directory")
}
for _, fi := range fis {
if fi.IsDir() {
continue
}
// Parse filename into integer.
shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
//AlwaysPrintf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name())
//v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name())
continue
}
sliceOfShards = append(sliceOfShards, shard)
}
return
}
func (tx *RoaringTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE
// the transaction Commits or Rollsback.
func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
b, err := tx.bitmap(index, field, view, shard)
panicOn(err)
return b.Iterator()
}
// ImportRoaringBits return values changed and rowSet will be inaccurate if
// the data []byte is supplied. This mimics the traditional roaring-per-file
// and should be faster.
func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
f, err := tx.getFragment(index, field, view, shard)
if err != nil {
return 0, nil, err
}
if len(data) > 0 {
// changed and rowSet are ignored anyway when len(data) > 0;
// when we are called from fragment.fillFragmentFromArchive()
// which is the only place the data []byte is supplied.
// blueGreenTx also turns off the checks in this case.
return 0, nil, f.readStorageFromArchive(bytes.NewBuffer(data))
}
changed, rowSet, err = f.storage.ImportRoaringRawIterator(rit, clear, true, rowSize)
return
}
func (tx *RoaringTx) Readonly() bool {
return !tx.write
}
func (tx *RoaringTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
frag, err := tx.getFragment(index, field, view, shard)
panicOn(err)
frag.incrementOpN(changedN)
}
// Rollback is a no-op as Roaring does not support transactions.
func (tx *RoaringTx) Rollback() {}
// Commit is a no-op as Roaring does not support transactions.
func (tx *RoaringTx) Commit() error {
return nil
}
func (tx *RoaringTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
return tx.bitmap(index, field, view, shard)
}
func (tx *RoaringTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, err
}
return b.Containers.Get(key), nil
}
func (tx *RoaringTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
b.Containers.Put(key, c)
return nil
}
func (tx *RoaringTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
b.Containers.Remove(key)
return nil
}
func (tx *RoaringTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
if !batched {
changed, err := b.Add(a...)
if changed {
return 1, err
}
return 0, err
}
// Note: do not replace b.AddN() with b.DirectAddN().
// DirectAddN() does not do op-log operations inside roaring
// This creates a problem because RoaringTx needs the op-log
// to know when to flush the fragment to disk.
count, err := b.AddN(a...) // AddN does oplog batches. needed to keep op-log up to date.
return count, err
}
func (tx *RoaringTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
changed, err := b.Remove(a...) // green TestFragment_Bug_Q2DoubleDelete
panicOn(err)
if changed {
return 1, err
} else {
return 0, err
}
// Note: don't replace b.Remove(a...) with b.RemoveN(a...) or
// with b.DirectRemoveN(a...). If you do, you'll see
// TestFragment_Bug_Q2DoubleDelete go red.
}
func (tx *RoaringTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return false, err
}
return b.Contains(v), nil
}
func (tx *RoaringTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, false, err
}
citer, found = b.Containers.Iterator(key)
return citer, found, nil
}
func (tx *RoaringTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
return b.ForEach(fn)
}
func (tx *RoaringTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
return b.ForEachRange(start, end, fn)
}
func (tx *RoaringTx) Count(index, field, view string, shard uint64) (uint64, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
return b.Count(), nil
}
func (tx *RoaringTx) Max(index, field, view string, shard uint64) (uint64, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
return b.Max(), nil
}
func (tx *RoaringTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, false, err
}
v, ok := b.Min()
return v, ok, nil
}
func (tx *RoaringTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
b.UnionInPlace(others...)
return nil
}
func (tx *RoaringTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
return b.CountRange(start, end), nil
}
func (tx *RoaringTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, err
}
return b.OffsetRange(offset, start, end), nil
}
// getFragment is used by IncrementOpN() and by bitmap()
func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*fragment, error) {
// If a fragment is attached, always use it. Since it was set at Tx creation,
// it is highly likely to be correct.
if tx.fragment != nil {
// but still a basic sanity check.
if tx.fragment.index != index ||
tx.fragment.field != field ||
tx.fragment.view != view ||
tx.fragment.shard != shard {
panic(fmt.Sprintf("different fragment cached vs requested. tx.fragment='%#v', index='%v', field='%v'; view='%v'; shard='%v'", tx.fragment, index, field, view, shard))
}
return tx.fragment, nil
}
// If a field is attached, start from there.
// Otherwise look up the field from the index.
f := tx.Field
if f == nil {
// we cannot assume that the tx.Index that we "started" on is the same
// as the index we are being queried; it might be foreign: TestExecutor_ForeignIndex
// So go through the holder
idx := tx.Index.holder.Index(index)
if idx == nil {
// only thing we can try is the cached index, and hope we aren't being asked for a foreign index.
f = tx.Index.Field(field)
if f == nil {
return nil, ErrFieldNotFound
}
} else {
if f = idx.Field(field); f == nil {
return nil, ErrFieldNotFound
}
}
}
// INVAR: f is not nil.
v := f.view(view)
if v == nil {
return nil, errors.Errorf("view not found: %q", view)
}
frag := v.Fragment(shard)
if frag == nil {
return nil, fmt.Errorf("fragment not found: %q / %q / %d", field, view, shard)
}
// Note: we cannot cache frag into tx.fragment.
// Empirically, it breaks 245 top-level pilosa tests.
// tx.fragment = frag // breaks the world.
return frag, nil
}
func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
frag, err := tx.getFragment(index, field, view, shard)
if err != nil {
return nil, err
}
return frag.storage, nil
}
type RoaringStore struct{}
func NewRoaringStore() *RoaringStore {
return &RoaringStore{}
}
func (db *RoaringStore) Close() error {
return nil
}
func (db *RoaringStore) DeleteField(index, field, fieldPath string) error {
// under blue-green badger_roaring, the directory will not be found, b/c badger will have
// already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs:
// "If the path does not exist, RemoveAll returns nil (no error)"
err := os.RemoveAll(fieldPath)
if err != nil {
return errors.Wrap(err, "removing directory")
}
return nil
}
// frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
fragment, ok := frag.(*fragment)
if !ok {
return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag)
}
// Close data files before deletion.
if err := fragment.Close(); err != nil {
return errors.Wrap(err, "closing fragment")
}
// Delete fragment file.
if err := os.Remove(fragment.path); err != nil {
return errors.Wrap(err, "deleting fragment file")
}
// Delete fragment cache file.
if err := os.Remove(fragment.cachePath()); err != nil {
return errors.Wrap(err, fmt.Sprintf("no cache file to delete for shard %d", fragment.shard))
}
return nil
}
func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
file, err := os.Open(fragmentPathForRoaring) // open the fragment file
if err != nil {
return nil, -1, err
}
fi, err := file.Stat()
if err != nil {
return nil, -1, errors.Wrap(err, "statting")
}
sz = fi.Size()
r = file
return
}

2
server.go
View file

@ -29,8 +29,6 @@ import (
uuid "github.com/satori/go.uuid"
// extensions pulls in some extensions depending on build tags
_ "github.com/pilosa/pilosa/v2/extensions"
"github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"

85
server/handler_test.go
View file

@ -179,6 +179,17 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatal(err)
}
defer tx0.Rollback()
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx0, 0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
if err := tx0.Commit(); err != nil {
t.Fatal(err)
}
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
tx1, err := holder.BeginTx(true, i1.Index)
@ -186,24 +197,11 @@ func TestHandler_Endpoints(t *testing.T) {
t.Fatal(err)
}
defer tx1.Rollback()
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx0, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx1, 0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
if err := tx0.Commit(); err != nil {
t.Fatal(err)
}
if err := tx1.Commit(); err != nil {
t.Fatal(err)
}
@ -673,11 +671,13 @@ func TestHandler_Endpoints(t *testing.T) {
if field == nil {
t.Fatalf("field not found: %s", fieldName)
}
if !reflect.DeepEqual(pql.NewDecimal(math.MinInt64, 0), field.Options.Min) {
t.Fatalf("field min %d != %d", int64(math.MinInt64), field.Options.Min)
}
if !reflect.DeepEqual(pql.NewDecimal(math.MaxInt64, 0), field.Options.Max) {
t.Fatalf("field max %d != %d", int64(math.MaxInt64), field.Options.Max)
if field != nil { // happy linter
if !reflect.DeepEqual(pql.NewDecimal(math.MinInt64, 0), field.Options.Min) {
t.Fatalf("field min %d != %d", int64(math.MinInt64), field.Options.Min)
}
if !reflect.DeepEqual(pql.NewDecimal(math.MaxInt64, 0), field.Options.Max) {
t.Fatalf("field max %d != %d", int64(math.MaxInt64), field.Options.Max)
}
}
})
@ -702,11 +702,13 @@ func TestHandler_Endpoints(t *testing.T) {
if field == nil {
t.Fatalf("field not found: %s", fieldName)
}
if !reflect.DeepEqual(pql.NewDecimal(math.MinInt64, 0), field.Options.Min) {
t.Fatalf("field min %d != %d", int64(math.MinInt64), field.Options.Min)
}
if !reflect.DeepEqual(pql.NewDecimal(1, -1), field.Options.Max) {
t.Fatalf("field max %d != %d", 10, field.Options.Max)
if field != nil { // happy linter
if !reflect.DeepEqual(pql.NewDecimal(math.MinInt64, 0), field.Options.Min) {
t.Fatalf("field min %d != %d", int64(math.MinInt64), field.Options.Min)
}
if !reflect.DeepEqual(pql.NewDecimal(1, -1), field.Options.Max) {
t.Fatalf("field max %d != %d", 10, field.Options.Max)
}
}
})
@ -731,11 +733,13 @@ func TestHandler_Endpoints(t *testing.T) {
if field == nil {
t.Fatalf("field not found: %s", fieldName)
}
if !reflect.DeepEqual(pql.NewDecimal(-1, -1), field.Options.Min) {
t.Fatalf("field min %d != %d", 10, field.Options.Min)
}
if !reflect.DeepEqual(pql.NewDecimal(math.MaxInt64, 0), field.Options.Max) {
t.Fatalf("field max %d != %d", int64(math.MaxInt64), field.Options.Max)
if field != nil { // happy linter
if !reflect.DeepEqual(pql.NewDecimal(-1, -1), field.Options.Min) {
t.Fatalf("field min %d != %d", 10, field.Options.Min)
}
if !reflect.DeepEqual(pql.NewDecimal(math.MaxInt64, 0), field.Options.Max) {
t.Fatalf("field max %d != %d", int64(math.MaxInt64), field.Options.Max)
}
}
})
@ -770,11 +774,13 @@ func TestHandler_Endpoints(t *testing.T) {
if field == nil {
t.Fatalf("field not found: %s", fieldName)
}
if !reflect.DeepEqual(pql.NewDecimal(math.MinInt64, 0), field.Options.Min) {
t.Fatalf("field min %d != %d", int64(math.MinInt64), field.Options.Min)
}
if !reflect.DeepEqual(pql.NewDecimal(math.MaxInt64, 0), field.Options.Max) {
t.Fatalf("field max %d != %d", int64(math.MaxInt64), field.Options.Max)
if field != nil { // happy linter
if !reflect.DeepEqual(pql.NewDecimal(math.MinInt64, 0), field.Options.Min) {
t.Fatalf("field min %d != %d", int64(math.MinInt64), field.Options.Min)
}
if !reflect.DeepEqual(pql.NewDecimal(math.MaxInt64, 0), field.Options.Max) {
t.Fatalf("field max %d != %d", int64(math.MaxInt64), field.Options.Max)
}
}
})
@ -800,11 +806,13 @@ func TestHandler_Endpoints(t *testing.T) {
if field == nil {
t.Fatalf("field not found: %s", fieldName)
}
if !reflect.DeepEqual(pql.NewDecimal(math.MinInt64, 1), field.Options.Min) {
t.Fatalf("field min %d != %d", pql.NewDecimal(math.MinInt64, 1), field.Options.Min)
}
if !reflect.DeepEqual(pql.NewDecimal(105, 1), field.Options.Max) {
t.Fatalf("field max %s != %d", pql.NewDecimal(105, 1), field.Options.Max)
if field != nil { // happy linter
if !reflect.DeepEqual(pql.NewDecimal(math.MinInt64, 1), field.Options.Min) {
t.Fatalf("field min %d != %d", pql.NewDecimal(math.MinInt64, 1), field.Options.Min)
}
if !reflect.DeepEqual(pql.NewDecimal(105, 1), field.Options.Max) {
t.Fatalf("field max %s != %d", pql.NewDecimal(105, 1), field.Options.Max)
}
}
})
@ -1054,6 +1062,7 @@ func TestHandler_Endpoints(t *testing.T) {
})
t.Run("index handlers", func(t *testing.T) {
// create index
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/idx1", strings.NewReader(""))

2
server_internal_test.go
View file

@ -23,6 +23,8 @@ import (
// Ensure the file handle count is working
func TestCountOpenFiles(t *testing.T) {
roaringOnlyTest(t)
// Windows is not supported yet
if runtime.GOOS == "windows" {
t.Skip("Skipping unsupported countOpenFiles test on Windows.")

1
translator_test.go
View file

@ -289,7 +289,6 @@ func TestTranslation_Reset(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if _, err := node0.API.TranslateKeys(ctx, bytes.NewReader(reqBody)); err != nil {
t.Fatal(err)
}

830
tx.go
View file

@ -15,18 +15,9 @@
package pilosa
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"github.com/pilosa/pilosa/v2/rbf"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
// batch operations want Tx.Add(batched=doBatch), while bit-at-a-time want Tx.Add(batched=!doBatched)
@ -240,824 +231,3 @@ type RawRoaringData struct {
func (rr *RawRoaringData) Iterator() (roaring.RoaringIterator, error) {
return roaring.NewRoaringIterator(rr.data)
}
// MultiTx implements the transaction interface to combine multiple transactions.
type MultiTx struct {
mu sync.Mutex
writable bool
holder *Holder
index *Index
txs map[multiTxKey]Tx
}
// NewMultiTx returns a new instance of MultiTx for a Holder.
func NewMultiTx(writable bool, holder *Holder) *MultiTx {
return &MultiTx{
writable: writable,
holder: holder,
txs: make(map[multiTxKey]Tx),
}
}
// NewMultiTxWithIndex returns a new instance of MultiTx for a single index.
func NewMultiTxWithIndex(writable bool, index *Index) *MultiTx {
return &MultiTx{
writable: writable,
index: index,
txs: make(map[multiTxKey]Tx),
}
}
var _ Tx = (*MultiTx)(nil)
func (mtx *MultiTx) Type() string {
return RoaringTxn
}
// debugging, what does this Tx see as its database?
func (mtx *MultiTx) Dump() {
mtx.mu.Lock()
defer mtx.mu.Unlock()
if len(mtx.txs) == 0 {
return
}
for _, tx := range mtx.txs {
tx.Dump()
return
}
}
func (mtx *MultiTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
tx, err := mtx.txNoShard(index)
panicOn(err)
return tx.SliceOfShards(index, field, view, optionalViewPath)
}
func (mtx *MultiTx) UseRowCache() bool {
return true
}
func (mtx *MultiTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
}
func (mtx *MultiTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.NewTxIterator(index, field, view, shard)
}
// Readonly is true if the transaction is not read-and-write, but only doing reads.
func (mtx *MultiTx) Readonly() bool {
return !mtx.writable
}
func (mtx *MultiTx) Pointer() string {
return fmt.Sprintf("%p", mtx)
}
func (mtx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
}
func (mtx *MultiTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
tx, err := mtx.tx(index, shard)
panicOn(err)
tx.IncrementOpN(index, field, view, shard, changedN)
}
// Rollback rolls back all underlying transactions.
func (mtx *MultiTx) Rollback() {
for _, tx := range mtx.txs {
tx.Rollback()
}
}
// Commit commits all underlying transactions.
func (mtx *MultiTx) Commit() (err error) {
for _, tx := range mtx.txs {
if e := tx.Commit(); e != nil && err == nil {
err = e
}
}
return err
}
func (mtx *MultiTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.RoaringBitmap(index, field, view, shard)
}
func (mtx *MultiTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.Container(index, field, view, shard, key)
}
func (mtx *MultiTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.PutContainer(index, field, view, shard, key, c)
}
func (mtx *MultiTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.RemoveContainer(index, field, view, shard, key)
}
func (mtx *MultiTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Add(index, field, view, shard, batched, a...)
}
func (mtx *MultiTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Remove(index, field, view, shard, a...)
}
func (mtx *MultiTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return false, err
}
return tx.Contains(index, field, view, shard, v)
}
func (mtx *MultiTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, false, err
}
return tx.ContainerIterator(index, field, view, shard, key)
}
func (mtx *MultiTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.ForEach(index, field, view, shard, fn)
}
func (mtx *MultiTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.ForEachRange(index, field, view, shard, start, end, fn)
}
func (mtx *MultiTx) Count(index, field, view string, shard uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Count(index, field, view, shard)
}
func (mtx *MultiTx) Max(index, field, view string, shard uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.Max(index, field, view, shard)
}
func (mtx *MultiTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, false, err
}
return tx.Min(index, field, view, shard)
}
func (mtx *MultiTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
tx, err := mtx.tx(index, shard)
if err != nil {
return err
}
return tx.UnionInPlace(index, field, view, shard, others...)
}
func (mtx *MultiTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return 0, err
}
return tx.CountRange(index, field, view, shard, start, end)
}
func (mtx *MultiTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return nil, err
}
return tx.OffsetRange(index, field, view, shard, offset, start, end)
}
// tx returns a transaction by index/shard. Reuses transaction if already open.
// Otherwise begins a new transaction.
func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
mkey := multiTxKey{index: index, shard: shard, write: mtx.writable}
// Lookup transaction from cache.
tx := mtx.txs[mkey]
if tx != nil {
return tx, nil
}
// If transaction doesn't exist, lookup the index.
idx := mtx.index
if mtx.holder != nil {
if idx = mtx.holder.Index(index); idx == nil {
return nil, ErrIndexNotFound
}
}
// Begin tranaction & cache it.
if tx, err = idx.BeginTx(mtx.writable, shard); err != nil {
return nil, err
}
mtx.txs[mkey] = tx
return tx, nil
}
// version of the above for SliceOfShards(), where we don't have a shard.
func (mtx *MultiTx) txNoShard(index string) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
// Lookup transaction from cache.
for _, tx := range mtx.txs {
if tx.(*RoaringTx).Index.name == index {
return tx, nil
}
}
panic(fmt.Sprintf("no prior RoaringTx available, looking up index='%v'", index))
}
type multiTxKey struct {
index string
shard uint64
write bool
}
// RoaringTx represents a fake transaction object for Roaring storage.
type RoaringTx struct {
write bool
Index *Index
Field *Field
fragment *fragment
}
func (tx *RoaringTx) Type() string {
return RoaringTxn
}
func (tx *RoaringTx) Dump() {
fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys())
}
func (tx *RoaringTx) UseRowCache() bool {
return true
}
func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
// SliceOfShards is based on view.openFragments()
file, err := os.Open(filepath.Join(optionalViewPath, "fragments"))
if os.IsNotExist(err) {
return
} else if err != nil {
return nil, errors.Wrap(err, "opening fragments directory")
}
defer file.Close()
fis, err := file.Readdir(0)
if err != nil {
return nil, errors.Wrap(err, "reading fragments directory")
}
for _, fi := range fis {
if fi.IsDir() {
continue
}
// Parse filename into integer.
shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
//AlwaysPrintf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name())
//v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name())
continue
}
sliceOfShards = append(sliceOfShards, shard)
}
return
}
func (tx *RoaringTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE
// the transaction Commits or Rollsback.
func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
b, err := tx.bitmap(index, field, view, shard)
panicOn(err)
return b.Iterator()
}
// ImportRoaringBits return values changed and rowSet will be inaccurate if
// the data []byte is supplied. This mimics the traditional roaring-per-file
// and should be faster.
func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
f, err := tx.getFragment(index, field, view, shard)
if err != nil {
return 0, nil, err
}
if len(data) > 0 {
// changed and rowSet are ignored anyway when len(data) > 0;
// when we are called from fragment.fillFragmentFromArchive()
// which is the only place the data []byte is supplied.
// blueGreenTx also turns off the checks in this case.
return 0, nil, f.readStorageFromArchive(bytes.NewBuffer(data))
}
changed, rowSet, err = f.storage.ImportRoaringRawIterator(rit, clear, true, rowSize)
return
}
func (tx *RoaringTx) Readonly() bool {
return !tx.write
}
func (tx *RoaringTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
frag, err := tx.getFragment(index, field, view, shard)
panicOn(err)
frag.incrementOpN(changedN)
}
// Rollback is a no-op as Roaring does not support transactions.
func (tx *RoaringTx) Rollback() {}
// Commit is a no-op as Roaring does not support transactions.
func (tx *RoaringTx) Commit() error {
return nil
}
func (tx *RoaringTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
return tx.bitmap(index, field, view, shard)
}
func (tx *RoaringTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, err
}
return b.Containers.Get(key), nil
}
func (tx *RoaringTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
b.Containers.Put(key, c)
return nil
}
func (tx *RoaringTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
b.Containers.Remove(key)
return nil
}
func (tx *RoaringTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
if !batched {
changed, err := b.Add(a...)
if changed {
return 1, err
}
return 0, err
}
// Note: do not replace b.AddN() with b.DirectAddN().
// DirectAddN() does not do op-log operations inside roaring
// This creates a problem because RoaringTx needs the op-log
// to know when to flush the fragment to disk.
count, err := b.AddN(a...) // AddN does oplog batches. needed to keep op-log up to date.
return count, err
}
func (tx *RoaringTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
changed, err := b.Remove(a...) // green TestFragment_Bug_Q2DoubleDelete
panicOn(err)
if changed {
return 1, err
} else {
return 0, err
}
// Note: don't replace b.Remove(a...) with b.RemoveN(a...) or
// with b.DirectRemoveN(a...). If you do, you'll see
// TestFragment_Bug_Q2DoubleDelete go red.
}
func (tx *RoaringTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return false, err
}
return b.Contains(v), nil
}
func (tx *RoaringTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, false, err
}
citer, found = b.Containers.Iterator(key)
return citer, found, nil
}
func (tx *RoaringTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
return b.ForEach(fn)
}
func (tx *RoaringTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
return b.ForEachRange(start, end, fn)
}
func (tx *RoaringTx) Count(index, field, view string, shard uint64) (uint64, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
return b.Count(), nil
}
func (tx *RoaringTx) Max(index, field, view string, shard uint64) (uint64, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
return b.Max(), nil
}
func (tx *RoaringTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, false, err
}
v, ok := b.Min()
return v, ok, nil
}
func (tx *RoaringTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
b.UnionInPlace(others...)
return nil
}
func (tx *RoaringTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
return b.CountRange(start, end), nil
}
func (tx *RoaringTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, err
}
return b.OffsetRange(offset, start, end), nil
}
// getFragment is used by IncrementOpN() and by bitmap()
func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*fragment, error) {
// If a fragment is attached, always use it. Since it was set at Tx creation,
// it is highly likely to be correct.
if tx.fragment != nil {
// but still a basic sanity check.
if tx.fragment.index != index ||
tx.fragment.field != field ||
tx.fragment.view != view ||
tx.fragment.shard != shard {
panic(fmt.Sprintf("different fragment cached vs requested. tx.fragment='%#v', index='%v', field='%v'; view='%v'; shard='%v'", tx.fragment, index, field, view, shard))
}
return tx.fragment, nil
}
// If a field is attached, start from there.
// Otherwise look up the field from the index.
f := tx.Field
if f == nil {
// we cannot assume that the tx.Index that we "started" on is the same
// as the index we are being queried; it might be foreign: TestExecutor_ForeignIndex
// So go through the holder
idx := tx.Index.holder.Index(index)
if idx == nil {
// only thing we can try is the cached index, and hope we aren't being asked for a foreign index.
f = tx.Index.Field(field)
if f == nil {
return nil, ErrFieldNotFound
}
} else {
if f = idx.Field(field); f == nil {
return nil, ErrFieldNotFound
}
}
}
// INVAR: f is not nil.
v := f.view(view)
if v == nil {
return nil, errors.Errorf("view not found: %q", view)
}
frag := v.Fragment(shard)
if frag == nil {
return nil, fmt.Errorf("fragment not found: %q / %q / %d", field, view, shard)
//panic(fmt.Sprintf("fragment not found: %q / %q / %d", field, view, shard))
}
// Note: we cannot cache frag into tx.fragment.
// Empirically, it breaks 245 top-level pilosa tests.
// tx.fragment = frag // breaks the world.
return frag, nil
}
func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
frag, err := tx.getFragment(index, field, view, shard)
if err != nil {
return nil, err
}
return frag.storage, nil
}
type RoaringStore struct{}
func NewRoaringStore() *RoaringStore {
return &RoaringStore{}
}
func (db *RoaringStore) Close() error {
return nil
}
func (db *RoaringStore) DeleteField(index, field, fieldPath string) error {
// under blue-green badger_roaring, the directory will not be found, b/c badger will have
// already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs:
// "If the path does not exist, RemoveAll returns nil (no error)"
err := os.RemoveAll(fieldPath)
if err != nil {
return errors.Wrap(err, "removing directory")
}
return nil
}
// frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
fragment, ok := frag.(*fragment)
if !ok {
return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag)
}
// Close data files before deletion.
if err := fragment.Close(); err != nil {
return errors.Wrap(err, "closing fragment")
}
// Delete fragment file.
if err := os.Remove(fragment.path); err != nil {
return errors.Wrap(err, "deleting fragment file")
}
// Delete fragment cache file.
if err := os.Remove(fragment.cachePath()); err != nil {
return errors.Wrap(err, fmt.Sprintf("no cache file to delete for shard %d", fragment.shard))
}
return nil
}
func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
file, err := os.Open(fragmentPathForRoaring) // open the fragment file
if err != nil {
return nil, -1, err
}
fi, err := file.Stat()
if err != nil {
return nil, -1, errors.Wrap(err, "statting")
}
sz = fi.Size()
r = file
return
}
type RBFTx struct {
index string
tx *rbf.Tx
}
func (tx *RBFTx) Type() string {
return RBFTxn
}
func (tx *RBFTx) Rollback() {
tx.tx.Rollback()
}
func (tx *RBFTx) Commit() error {
return tx.tx.Commit()
}
func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
return tx.tx.RoaringBitmap(rbfName(field, view, shard))
}
func (tx *RBFTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
return tx.tx.Container(rbfName(field, view, shard), key)
}
func (tx *RBFTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
return tx.tx.PutContainer(rbfName(field, view, shard), key, c)
}
func (tx *RBFTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
return tx.tx.RemoveContainer(rbfName(field, view, shard), key)
}
func (tx *RBFTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
return tx.tx.Add(rbfName(field, view, shard), a...)
}
func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
return tx.tx.Remove(rbfName(field, view, shard), a...)
}
func (tx *RBFTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
return tx.tx.Contains(rbfName(field, view, shard), v)
}
func (tx *RBFTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
return tx.tx.ContainerIterator(rbfName(field, view, shard), key)
}
func (tx *RBFTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
return tx.tx.ForEach(rbfName(field, view, shard), fn)
}
func (tx *RBFTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
return tx.tx.ForEachRange(rbfName(field, view, shard), start, end, fn)
}
func (tx *RBFTx) Count(index, field, view string, shard uint64) (uint64, error) {
return tx.tx.Count(rbfName(field, view, shard))
}
func (tx *RBFTx) Max(index, field, view string, shard uint64) (uint64, error) {
return tx.tx.Max(rbfName(field, view, shard))
}
func (tx *RBFTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
return tx.tx.Min(rbfName(field, view, shard))
}
func (tx *RBFTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
return tx.tx.UnionInPlace(rbfName(field, view, shard), others...)
}
func (tx *RBFTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
return tx.tx.CountRange(rbfName(field, view, shard), start, end)
}
func (tx *RBFTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
return tx.tx.OffsetRange(rbfName(field, view, shard), offset, start, end)
}
func (tx *RBFTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {}
func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
return tx.tx.ImportRoaringBits(rbfName(field, view, shard), rit, clear, log, rowSize, data)
}
func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
panic("TODO: Implement RBFTx.RoaringBitmapReader()")
}
func (tx *RBFTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
prefix := rbfFieldViewPrefix(field, view)
names, err := tx.tx.BitmapNames()
if err != nil {
return nil, err
}
// Iterate over shard names and collect shards from matching field/view prefix.
for _, name := range names {
if !strings.HasPrefix(name, prefix) {
continue
}
s := strings.TrimPrefix(name, prefix)
shard, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return nil, errors.Wrap(err, "parse shard id from rbf key")
}
sliceOfShards = append(sliceOfShards, shard)
}
return sliceOfShards, nil
}
func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
b, err := tx.RoaringBitmap(index, field, view, shard)
panicOn(err)
return b.Iterator()
}
func (tx *RBFTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
func (tx *RBFTx) Dump() {
tx.tx.Dump(tx.index)
}
// Readonly is true if the transaction is not read-and-write, but only doing reads.
func (tx *RBFTx) Readonly() bool {
return !tx.tx.Writable()
}
func (tx *RBFTx) UseRowCache() bool {
return false
}
// rbfName returns a NULL-separated key used for identifying bitmap maps in RBF.
func rbfName(field, view string, shard uint64) string {
return fmt.Sprintf("%s\x00%s\x00%d", field, view, shard)
}
// rbfFieldPrefix returns a prefix for field keys in RBF.
func rbfFieldPrefix(field string) string {
return fmt.Sprintf("%s\x00", field)
}
// rbfFieldViewPrefix returns a NULL-separated prefix for keys in RBF.
func rbfFieldViewPrefix(field, view string) string {
return fmt.Sprintf("%s\x00%s\x00", field, view)
}

176
txfactory.go
View file

@ -23,8 +23,8 @@ import (
"syscall"
"text/tabwriter"
"github.com/pilosa/pilosa/v2/rbf"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/txpath"
"github.com/pkg/errors"
)
@ -32,6 +32,7 @@ import (
const (
RoaringTxn string = "roaring"
BadgerTxn string = "badger"
LmdbTxn string = "lmdb"
RBFTxn string = "rbf"
// A is listed first, B is second. blueGreenTx returns the B output.
BlueGreenBadgerRoaring string = "badger_roaring"
@ -72,10 +73,14 @@ type TxFactory struct {
badgerDB *BadgerDBWrapper
rbfDB *rbf.DB
rbfDB *RbfDBWrapper
roaringDB *RoaringStore
dbsClosed bool // idemopotent CloseDB()
//lmDB *LMDBWrapper
// could have more than one *Index, but for now keep it simple,
// and allow blueGreenTx to report badger contents via idx
idx *Index
@ -100,6 +105,8 @@ const (
blueGreenBadgerRBF txtype = 8
blueGreenRBFBadger txtype = 9
lmdbTxn txtype = 10
)
func (txf *TxFactory) NeedsSnapshot() bool {
@ -124,6 +131,8 @@ func (txf *TxFactory) NeedsSnapshot() bool {
return false
case blueGreenRBFBadger:
return false
case lmdbTxn:
return false
}
panic(fmt.Sprintf("unknown typeOfTx '%v'", txf.typeOfTx))
}
@ -148,16 +157,21 @@ func MustTxsrcToTxtype(txsrc string) txtype {
return blueGreenBadgerRBF
case BlueGreenRBFBadger: // "rbf_badger"
return blueGreenRBFBadger
case LmdbTxn:
return lmdbTxn
}
panic(fmt.Sprintf("unknown txsrc '%v'", txsrc))
}
// always store files in a subdir of dir. If we are having one
// NewTxFactory always opens an existing database. If you
// want to a fresh database, os.RemoveAll on dir/name ahead of time.
// We always store files in a subdir of dir. If we are having one
// database or many can depend on name.
func NewTxFactory(txsrc string, dir, name string, openExisting bool) (f *TxFactory, err error) {
func NewTxFactory(txsrc string, dir, name string) (f *TxFactory, err error) {
//vv("NewTxFactory called for txsrc '%v'; dir='%v'; name='%v'", txsrc, dir, name)
ty := MustTxsrcToTxtype(txsrc)
if ty < 1 || ty > 9 {
if ty < 1 || ty > 10 {
panic(fmt.Sprintf("invalid txtype '%v'", int(ty)))
}
@ -175,17 +189,11 @@ func NewTxFactory(txsrc string, dir, name string, openExisting bool) (f *TxFacto
// enables cross-index Tx, which are important and are tested for.
path := dir + sep + "honeyBadger"
if openExisting {
f.badgerDB, err = globalBadgerReg.openBadgerDBWrapper(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot open badger db. path='%v'", path))
}
} else {
f.badgerDB, err = globalBadgerReg.newBadgerDBWrapper(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot create new badger db. path='%v'", path))
}
f.badgerDB, err = globalBadgerReg.openBadgerDBWrapper(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot open badger db. path='%v'", path))
}
// electric-fence like finding of access to mmapped data beyond
// transaction end time.
f.badgerDB.doAllocZero = DetectMemAccessPastTx
@ -194,12 +202,25 @@ func NewTxFactory(txsrc string, dir, name string, openExisting bool) (f *TxFacto
switch ty {
case rbfTxn, blueGreenRBFRoaring, blueGreenRoaringRBF, blueGreenBadgerRBF, blueGreenRBFBadger:
f.rbfDB = rbf.NewDB(filepath.Join(dir, "db.rbf"))
if err := f.rbfDB.Open(); err != nil {
return nil, errors.Wrap(err, "cannot open rbf db")
path := dir + sep + "all-in-one-rbfdb"
f.rbfDB, err = globalRbfDBReg.openRbfDB(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot create new rbf db. path='%v'", path))
}
}
switch ty {
case lmdbTxn:
panic("lmdb is unfinished and relocated to the ldmb/ subdirectory for the moment.")
/*
path := dir + sep + "all-in-one"
f.lmDB, err = globalLMDBReg.newLMDBWrapper(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot create new lmdb db. path='%v'", path))
}
*/
}
return f, err
}
@ -224,7 +245,7 @@ func (f *TxFactory) DeleteIndex(name string) error {
case badgerTxn:
return f.badgerDB.DeleteIndex(name)
case rbfTxn:
panic("todo rbfTxn DeleteIndex(name)")
return f.rbfDB.DeleteIndex(name)
case blueGreenBadgerRoaring:
return f.badgerDB.DeleteIndex(name)
case blueGreenRoaringBadger:
@ -239,22 +260,10 @@ func (f *TxFactory) DeleteFieldFromStore(index, field, fieldPath string) error {
return f.roaringDB.DeleteField(index, field, fieldPath)
case badgerTxn:
return f.badgerDB.DeleteField(index, field, fieldPath)
//case lmdbTxn:
//return f.lmDB.DeleteField(index, field, fieldPath)
case rbfTxn:
if err := os.RemoveAll(fieldPath); err != nil {
return errors.Wrap(err, "removing directory")
}
tx, err := f.rbfDB.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
if err := tx.DeleteBitmapsWithPrefix(rbfFieldPrefix(field)); err != nil {
return err
}
return tx.Commit()
return f.rbfDB.DeleteField(index, field, fieldPath)
case blueGreenBadgerRoaring:
_ = f.badgerDB.DeleteField(index, field, fieldPath)
return f.roaringDB.DeleteField(index, field, fieldPath)
@ -272,16 +281,9 @@ func (f *TxFactory) DeleteFragmentFromStore(index, field, view string, shard uin
case badgerTxn:
return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
case rbfTxn:
tx, err := f.rbfDB.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
if err := tx.DeleteBitmapsWithPrefix(rbfFieldViewPrefix(field, view)); err != nil {
return err
}
return tx.Commit()
return f.rbfDB.DeleteFragment(index, field, view, shard, frag)
// case lmdbTxn:
// return f.lmDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenBadgerRoaring:
_ = f.badgerDB.DeleteFragment(index, field, view, shard, frag)
return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
@ -294,32 +296,38 @@ func (f *TxFactory) DeleteFragmentFromStore(index, field, view string, shard uin
}
func (f *TxFactory) CloseIndex(idx *Index) error {
// under roaring and all the new databases, this is a no-op.
return nil
}
func (f *TxFactory) CloseDB() error {
if f.dbsClosed {
return nil
}
f.dbsClosed = true
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return nil
case badgerTxn:
// note cannot actually close Badger here.
// causes problems b/c tries holder.DeleteIndex tries to delete the index after db is closed.
//return f.badgerDB.Close()
return nil
return f.badgerDB.Close()
case rbfTxn:
return f.rbfDB.Close()
case blueGreenBadgerRoaring:
return nil
return f.badgerDB.Close()
case blueGreenRoaringBadger:
return nil
return f.badgerDB.Close()
case blueGreenRBFRoaring:
_ = f.rbfDB.Close()
return nil
return f.rbfDB.Close()
case blueGreenRoaringRBF:
return f.rbfDB.Close()
case blueGreenBadgerRBF:
_ = f.badgerDB.Close()
return f.rbfDB.Close()
case blueGreenRBFBadger:
_ = f.rbfDB.Close()
return f.badgerDB.Close()
case lmdbTxn:
return nil
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
@ -334,52 +342,56 @@ func (f *TxFactory) NewTx(o Txo) Tx {
case roaringFragmentFilesTxn:
return &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
case badgerTxn:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
btx := f.badgerDB.NewBadgerTx(o.Write, indexName, o.Fragment)
return btx
case rbfTxn:
tx, err := f.rbfDB.Begin(o.Write)
tx, err := f.rbfDB.NewRBFTx(o.Write, indexName, o.Fragment)
panicOn(err)
if err != nil {
panic(err) // TODO: Add error return on NewTx()
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
}
return &RBFTx{tx: tx, index: indexName}
return tx
case lmdbTxn:
//return f.lmDB.newPoolTx(o.Write, indexName)
case blueGreenBadgerRoaring:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
btx := f.badgerDB.NewBadgerTx(o.Write, indexName, o.Fragment)
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(btx, rtx, f.idx)
case blueGreenRoaringBadger:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
btx := f.badgerDB.NewBadgerTx(o.Write, indexName, o.Fragment)
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(rtx, btx, f.idx)
case blueGreenBadgerRBF:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
rbftx, err := f.rbfDB.Begin(o.Write)
btx := f.badgerDB.NewBadgerTx(o.Write, indexName, o.Fragment)
rbftx, err := f.rbfDB.NewRBFTx(o.Write, indexName, o.Fragment)
if err != nil {
panic(errors.Wrap(err, "rbfDB.Begin transaction errored"))
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
}
return newBlueGreenTx(btx, &RBFTx{tx: rbftx, index: indexName}, f.idx)
return newBlueGreenTx(btx, rbftx, f.idx)
case blueGreenRBFBadger:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
rbftx, err := f.rbfDB.Begin(o.Write)
btx := f.badgerDB.NewBadgerTx(o.Write, indexName, o.Fragment)
rbftx, err := f.rbfDB.NewRBFTx(o.Write, indexName, o.Fragment)
if err != nil {
panic(errors.Wrap(err, "rbfDB.Begin transaction errored"))
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
}
return newBlueGreenTx(&RBFTx{tx: rbftx, index: indexName}, btx, f.idx)
return newBlueGreenTx(rbftx, btx, f.idx)
case blueGreenRBFRoaring:
rbftx, err := f.rbfDB.Begin(o.Write)
rbftx, err := f.rbfDB.NewRBFTx(o.Write, indexName, o.Fragment)
if err != nil {
panic(errors.Wrap(err, "rbfDB.Begin transaction errored"))
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
}
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(&RBFTx{tx: rbftx, index: indexName}, rtx, f.idx)
return newBlueGreenTx(rbftx, rtx, f.idx)
case blueGreenRoaringRBF:
rbftx, err := f.rbfDB.Begin(o.Write)
rbftx, err := f.rbfDB.NewRBFTx(o.Write, indexName, o.Fragment)
if err != nil {
panic(errors.Wrap(err, "rbfDB.Begin transaction errored"))
panic(errors.Wrap(err, "rbfDB.NewRBFTx transaction errored"))
}
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(rtx, &RBFTx{tx: rbftx, index: indexName}, f.idx)
return newBlueGreenTx(rtx, rbftx, f.idx)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
@ -406,6 +418,8 @@ func (ty txtype) String() string {
return "blueGreenBadgerRBF"
case blueGreenRBFBadger:
return "blueGreenRBFBadger"
case lmdbTxn:
return "lmdbTxn"
}
panic(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty)))
}
@ -550,7 +564,7 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard
srbm := bitmapAsString(rbm)
panicOn(err)
bkey := string(badgerKey(index, field, view, shard, ckey))
bkey := string(txpath.Key(index, field, view, shard, ckey))
r += fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, ct.N())
r += " ......." + srbm + "\n"
@ -619,16 +633,16 @@ var _ = fileSize // happy linter
func containerToBytes(ct *roaring.Container) []byte {
ty := roaring.ContainerType(ct)
switch ty {
case containerNil:
panic("nil container")
case containerArray:
case roaring.ContainerNil:
panic("nil roaring.Container")
case roaring.ContainerArray:
return fromArray16(roaring.AsArray(ct))
case containerBitmap:
case roaring.ContainerBitmap:
return fromArray64(roaring.AsBitmap(ct))
case containerRun:
case roaring.ContainerRun:
return fromInterval16(roaring.AsRuns(ct))
}
panic(fmt.Sprintf("unknown container type '%v'", int(ty)))
panic(fmt.Sprintf("unknown roaring.Container type '%v'", int(ty)))
}
type pointerContext struct {

157
txpath/txpath.go Normal file
View file

@ -0,0 +1,157 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package txpath consolidates in one place the use of keys to index into our
// various storage/txn back-ends. Databases badgerDB and rbfDB both use it,
// so that debug Dumps are comparable.
package txpath
import (
"bytes"
"fmt"
"strconv"
)
// Key produces the bytes that we use as a key to query the storage/tx engine.
// The roaringContainerKey argument is a container key into a roaring Container.
// Output examples:
//
// "idx:'i';fld:'f';vw:'standard';shd:'0';ckey@00000000000000000000" // smallest container-key
// "idx:'i';fld:'f';vw:'standard';shd:'0';ckey@18446744073709551615" // largest container-key (math.MaxUint64)
//
// NB must be kept in sync with Prefix() and KeyExtractContainerKey().
//
func Key(index, field, view string, shard uint64, roaringContainerKey uint64) []byte {
// The %020d which adds zero padding up to 20 runes is required to
// allow the textual sort to accurately
// reflect a numeric sort order. This is because, as a string,
// math.MaxUint64 is 20 bytes long.
// Example of such a Key with a container-key that is math.MaxUint64:
// ...........................................12345678901234567890
// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615
prefix := Prefix(index, field, view, shard)
ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
bkey := append(prefix, ckey...)
MustValidateKey(bkey)
return bkey
}
var ckeyPartExpected = []byte(";ckey@")
// MustValidatekey will panic on a bad Key with an informative message.
func MustValidateKey(bkey []byte) {
n := len(bkey)
if n < 56 {
panic(fmt.Sprintf("bkey too short min size is 56 but we see %v in '%v'", n, string(bkey)))
}
beforeCkey := bkey[n-26 : n-20]
if !bytes.Equal(beforeCkey, ckeyPartExpected) {
panic(fmt.Sprintf(`bkey did not have expected ";ckey@" at 26 bytes from the end of the bkey '%v'; instead had '%v'`, string(bkey), string(beforeCkey)))
}
}
func ShardFromKey(bkey []byte) (shard uint64) {
MustValidateKey(bkey)
n := len(bkey)
// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615 -> idx:'i';fld:'f';vw:'standard';shd:'1
by := bkey[:n-27]
beg := bytes.LastIndex(by, []byte("'"))
if beg == -1 {
panic(fmt.Sprintf("bad bkey='%v' did not have single quote to being shard decoding", string(bkey)))
}
parseMe := string(by[beg+1:])
shard, err := strconv.ParseUint(parseMe, 10, 64)
if err != nil {
panic(fmt.Sprintf("could not parse parseMe '%v' in strconv.ParseUint(), error: '%v'", parseMe, err))
}
return shard
}
func ShardFromPrefix(prefix []byte) (shard uint64) {
n := len(prefix)
// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@ -> idx:'i';fld:'f';vw:'standard';shd:'1
by := prefix[:n-7]
beg := bytes.LastIndex(by, []byte("'"))
if beg == -1 {
panic(fmt.Sprintf("bad prefix='%v' did not have single quote to being shard decoding", string(prefix)))
}
parseMe := string(by[beg+1:])
shard, err := strconv.ParseUint(parseMe, 10, 64)
if err != nil {
panic(fmt.Sprintf("could not parse parseMe '%v' in strconv.ParseUint(), error: '%v'", parseMe, err))
}
return shard
}
// KeyAndPrefix returns the equivalent of Key() and Prefix() calls.
func KeyAndPrefix(index, field, view string, shard uint64, roaringContainerKey uint64) (key, prefix []byte) {
prefix = Prefix(index, field, view, shard)
ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
bkey := append(prefix, ckey...)
MustValidateKey(bkey)
return bkey, prefix
}
var _ = KeyAndPrefix // keep linter happy
// KeyExtractContainerKey extracts the containerKey from bkey.
func KeyExtractContainerKey(bkey []byte) (containerKey uint64) {
MustValidateKey(bkey)
// The zero padding means that the container-key is always the last 20 bytes of the bkey.
//
// Be sure to catch the problematic case of a user passing in only a prefix. A prefix
// ends in 'key@' rather than a full key that has 'key@00000000000000000001' (for example)
// at the end. The ParseUint call below will fail in that case.
n := len(bkey)
if n < 20 {
panic(fmt.Sprintf("KeyExtractContainerKey() error: bad bkey '%v', too short!", string(bkey)))
}
last := bkey[n-20:] // Key() and Prefix() always return more than 20 rune []byte.
var err error
containerKey, err = strconv.ParseUint(string(last), 10, 64) // has to be the container key
if err != nil {
panic(fmt.Sprintf("KeyExtractContainerKey() error: bad bkey '%v', could not convert last 20 bytes ('%v') to a unit64: '%v'", string(bkey), string(last), err))
}
return
}
func AllShardPrefix(index, field, view string) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:", index, field, view))
}
// Prefix returns everything from Key up to and
// including the '@' fune in a Key. The prefix excludes the roaring container key itself.
// NB must be kept in sync with Key() and KeyExtractContainerKey().
func Prefix(index, field, view string, shard uint64) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@", index, field, view, shard))
}
// IndexOnlyPrefix returns a prefix suitable for DeleteIndex and a key-scan to
// remove all storage associated with one index.
//
// The full name of the index must be provided, no partial index names will work.
//
// The provided key is terminated by `';` and so DeleteIndex("i") will not delete the index "i2".
//
func IndexOnlyPrefix(indexName string) []byte {
return []byte(fmt.Sprintf("idx:'%v';", indexName))
}
// same for deleting a whole field.
func FieldPrefix(index, field string) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';", index, field))
}

89
txpath/txpath_test.go Normal file
View file

@ -0,0 +1,89 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package txpath
import (
"bytes"
"fmt"
"strconv"
"testing"
)
func Test_KeyPrefix(t *testing.T) {
// Prefix() must agree with Key(), but not have the key at the end.
// This is important for iteration over containers.
index, field, view, shard := "i", "f", "v", uint64(0)
// needle examples with the container-key extremes:
// "index:'i';field:'f';view:'v';shard:'0';key@00000000000000000000" // smallest
// "index:'i';field:'f';view:'v';shard:'0';key@18446744073709551615" // largest
needle := Key(index, field, view, shard, 0)
// prefix example: "index:'i';field:'f';view:'v';shard:'0';key@"
prefix := Prefix(index, field, view, shard)
if !bytes.HasPrefix(needle, prefix) {
panic(fmt.Sprintf("Prefix() output '%v'was not a prefix of Key() '%v'", string(needle), string(prefix)))
}
if len(prefix)+20 != len(needle) {
panic(fmt.Sprintf("Prefix() output '%v'was 20 characters shorter than Key() '%v'", string(needle), string(prefix)))
}
// validate assumption that KeyExtractContainerKey() makes about strconv.ParseUint() error reporting;
// for distinguishing prefixes from full keys. Even if the shard number is so large that the prefix
// starts with a legitimate decimal number.
shouldNotParse := "12345123451234';key@"
containerKey, err := strconv.ParseUint(shouldNotParse, 10, 64)
if err == nil {
panic(fmt.Sprintf("strconv.ParseUint should have returned an error parsing this string '%v'; instead we got '%v'", shouldNotParse, containerKey))
}
// verify panic on submitting a prefix
func() {
defer func() {
r := recover()
if r == nil {
panic(fmt.Sprintf("should have seen panic on call to KeyExtractContainerKey(prefix='%v')", prefix))
}
}()
KeyExtractContainerKey(prefix) // should panic.
}()
}
func Test_ShardFromKey(t *testing.T) {
if ShardFromKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615")) != 1 {
panic("problem")
}
if ShardFromKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'0';ckey@18446744073709551615")) != 0 {
panic("problem")
}
if ShardFromKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@18446744073709551615")) != 18446744073709551615 {
panic("problem")
}
func() {
defer func() {
r := recover()
if r == nil {
panic("should have panic-ed")
}
}()
// called for the panic of a short ckey, only 19 bytes instead of 20
ShardFromKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@1844674407370955161"))
}()
}

7
extensions/dummy.go → txpath/txprefix.go~
View file

@ -1,4 +1,4 @@
// Copyright 2019 Pilosa Corp.
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,7 +12,4 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// This package contains only things which are conditional on build
// tags.
package extensions
package txprefix

44
txpath/txprefix_test.go~ Normal file
View file

@ -0,0 +1,44 @@
package txprefix
func TestBadger_KeyPrefix(t *testing.T) {
// txprefix.Prefix() must agree with txprefix.Key(), but not have the key at the end.
// This is important for iteration over containers.
index, field, view, shard := "i", "f", "v", uint64(0)
// needle examples with the container-key extremes:
// "index:'i';field:'f';view:'v';shard:'0';key@00000000000000000000" // smallest
// "index:'i';field:'f';view:'v';shard:'0';key@18446744073709551615" // largest
needle := txprefix.Key(index, field, view, shard, 0)
// prefix example: "index:'i';field:'f';view:'v';shard:'0';key@"
prefix := txprefix.Prefix(index, field, view, shard)
if !bytes.HasPrefix(needle, prefix) {
panic(fmt.Sprintf("txprefix.Prefix() output '%v'was not a prefix of txprefix.Key() '%v'", string(needle), string(prefix)))
}
if len(prefix)+20 != len(needle) {
panic(fmt.Sprintf("txprefix.Prefix() output '%v'was 20 characters shorter than txprefix.Key() '%v'", string(needle), string(prefix)))
}
// validate assumption that txprefix.KeyExtractContainerKey() makes about strconv.ParseUint() error reporting;
// for distinguishing prefixes from full keys. Even if the shard number is so large that the prefix
// starts with a legitimate decimal number.
shouldNotParse := "12345123451234';key@"
containerKey, err := strconv.ParseUint(shouldNotParse, 10, 64)
if err == nil {
panic(fmt.Sprintf("strconv.ParseUint should have returned an error parsing this string '%v'; instead we got '%v'", shouldNotParse, containerKey))
}
// verify panic on submitting a prefix
func() {
defer func() {
r := recover()
if r == nil {
panic(fmt.Sprintf("should have seen panic on call to txprefix.KeyExtractContainerKey(prefix='%v')", prefix))
}
}()
txprefix.KeyExtractContainerKey(prefix) // should panic.
}()
}