scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 09:05:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Tx integration milestone

Browse source 
a) All tests green under -race for both PILOSA_TXSRC=roaring and PILOSA_TXSRC=badger.

b) Distinct is merged back into mainline pilosa.

Seebs notes on the Distinct work:

merge Distinct plugin back into main source tree, convert to Tx

We drop all references to the Preemptively Deprecated Don't You Dare
Use This extension interface, and move the one and only extension we had
(Distinct) into the main executor.

Also this fixes an arguable bug, which is that Container.AsBitmap()
would panic on a nil parameter, but it should have returned an empty
bitmap, because a nil *Ccontainer is a valid empty container. This
simplifies logic significantly in Distinct.

Fixes #569 #570 #571 #572 #573 #584 #585
This commit is contained in:
Jason Aten 2020-07-21 11:36:09 -04:00
parent efc90a6d36
commit ac7be132ef
35 changed files with 2795 additions and 1057 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
Makefile
View file

@ -16,16 +16,13 @@ RELEASE_ENABLED = $(subst 0,,$(RELEASE))
NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" )
BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
BUILD_TAGS += shardwidth$(SHARD_WIDTH)
BUILD_TAGS += $(foreach p,$(PLUGINS),plugin$(p))
define LICENSE_HASH_CODE
head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
endef
LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
PLUGINS=distinct
export GO111MODULE=on
export GOPRIVATE=github.com/molecula
export PLUGINS
# Run tests and compile Pilosa
default: test build
@ -97,7 +94,7 @@ clustertests: vendor
# Like clustertests, but rebuilds all images.
clustertests-build: vendor
docker-compose -f $(DOCKER_COMPOSE) down
docker-compose -f $(DOCKER_COMPOSE) down -v
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
# Create prerelease builds
@ -152,25 +149,80 @@ docker-test:
# run top tests, not subdirs. print summary red/green after.
# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
topt:
mv log.topt.roar log.topt.roar.prev || true
go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.roar
@echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
@echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l
topt-badger:
mv log.topt.badger log.topt.badger.prev || true
PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger
@echo " log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l
@echo " log.topt.badger red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l
topt-rb:
mv log.topt.roaring_badger log.topt.roaring_badger.prev || true
PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger
@echo " log.topt.roaring_badger green: \c"; cat log.topt.roaring_badger | grep PASS |wc -l
@echo " log.topt.roaring_badger red: \c"; cat log.topt.roaring_badger | grep '\-\-\- FAIL' |wc -l
topt-badger-race:
mv log.topt.badger-race log.topt.badger-race.prev || true
PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger-race
@echo " log.topt.badger-race green: \c"; cat log.topt.badger-race | grep PASS |wc -l
@echo " log.topt.badger-race red: \c"; cat log.topt.badger-race | grep '\-\-\- FAIL' |wc -l
topt-rbf:
mv log.topt.rbf log.topt.rbf.prev || true
PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.rbf
@echo " log.topt.rbf green: \c"; cat log.topt.rbf | grep PASS |wc -l
@echo " log.topt.rbf red: \c"; cat log.topt.rbf | grep '\-\-\- FAIL' |wc -l
topt-race:
mv log.topt.race log.topt.race.prev || true
go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.race
@echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
@echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l
# blue-green checks. These run two different storage engines (rbf, roaring, or badger)
# and compare each transaction for a result.
bg-br:
mv log.bg.bg_roar log.bg.bg_roar.prev || true
PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg.bg_roar
@echo " log.bg.bg_roar green: \c"; cat log.bg.bg_roar | grep PASS |wc -l
@echo " log.bg.bg_roar red: \c"; cat log.bg.bg_roar | grep '\-\-\- FAIL' |wc -l
bg-rb:
mv log.bg.roar_bg log.bg.roar_bg.prev || true
PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg.roar_bg
@echo " log.bg.roar_bg green: \c"; cat log.bg.roar_bg | grep PASS |wc -l
@echo " log.bg.roar_bg red: \c"; cat log.bg.roar_bg | grep '\-\-\- FAIL' |wc -l
bg-fr:
mv log.bg.rbf_roar log.bg.rbf_roar.prev || true
PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg.rbf_roar
@echo " log.bg.rbf_roar green: \c"; cat log.bg.rbf_roar | grep PASS |wc -l
@echo " log.bg.rbf_roar red: \c"; cat log.bg.rbf_roar | grep '\-\-\- FAIL' |wc -l
bg-rf:
mv log.bg.roar_rbf log.bg.roar_rbf.prev || true
PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg.roar_rbf
@echo " log.bg.roar_rbf green: \c"; cat log.bg.roar_rbf | grep PASS |wc -l
@echo " log.bg.roar_rbf red: \c"; cat log.bg.roar_rbf | grep '\-\-\- FAIL' |wc -l
bg-fb:
mv log.bg.rbf_badger log.bg.rbf_badger.prev || true
PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg.rbf_badger
@echo " log.bg.rbf_badger green: \c"; cat log.bg.rbf_badger | grep PASS |wc -l
@echo " log.bg.rbf_badger red: \c"; cat log.bg.rbf_badger | grep '\-\-\- FAIL' |wc -l
bg-bf:
mv log.bg.badger_rbf log.bg.badger_rbf.prev || true
PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg.badger_rbf
@echo " log.bg.badger_rbf green: \c"; cat log.bg.badger_rbf | grep PASS |wc -l
@echo " log.bg.badger_rbf red: \c"; cat log.bg.badger_rbf | grep '\-\-\- FAIL' |wc -l
# Run golangci-lint
golangci-lint: require-golangci-lint
golangci-lint run --skip-files '.*\.peg\.go'

9
api.go
View file

@ -371,16 +371,9 @@ func importWorker(importWork chan importJob) {
var doClear bool
switch doAction {
case RequestActionOverwrite:
// TODO(jea): the question here is, why are we commiting this separately from j.tx?
// why doesn't j.tx suffice? It doesn't but why/which is correct?
tx := j.field.holder.indexes[j.field.index].Txf.NewTx(Txo{Write: true, Field: j.field})
defer tx.Rollback()
if err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block); err != nil {
if err := j.field.importRoaringOverwrite(j.ctx, j.tx, viewData, j.shard, viewName, j.req.Block); err != nil {
return errors.Wrap(err, "importing roaring as overwrite")
}
if err := tx.Commit(); err != nil {
return errors.Wrap(err, "commit of importing roaring as overwrite")
}
case RequestActionClear:
doClear = true
fallthrough

418
badger.go
View file

@ -15,8 +15,9 @@
package pilosa
import (
"errors"
"bytes"
"fmt"
"io"
"io/ioutil"
"log"
"os"
@ -29,7 +30,9 @@ import (
"unsafe"
badger "github.com/dgraph-io/badger/v2"
badgeroptions "github.com/dgraph-io/badger/v2/options"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
// TODO: is there a more optimal time to do badger garbage collection?
@ -189,15 +192,70 @@ func (l *BadgerLog) Debugf(f string, v ...interface{}) {
l.Printf("DEBUG: "+f, v...)
}
// badgerRegistrar facilitates shutdown
// of all the badger databases started under
// tests. Its needed because most tests don't cleanup
// the *Index(es) they create. But we still
// want to shutdown badgerDB goroutines
// after tests run.
//
// It also allows opening the same path twice to
// result in sharing the same open database handle, and
// thus the same transactional guarantees.
//
type badgerRegistrar struct {
mu sync.Mutex
mp map[*BadgerDBWrapper]bool
path2db map[string]*BadgerDBWrapper
}
var globalBadgerReg *badgerRegistrar = newBadgerTestRegistrar()
func newBadgerTestRegistrar() *badgerRegistrar {
return &badgerRegistrar{
mp: make(map[*BadgerDBWrapper]bool),
path2db: make(map[string]*BadgerDBWrapper),
}
}
// register each badger created under tests, so we
// can clean them up. This is called by openBadgerDBWrapper() while
// holding the r.mu.Lock, since it needs to atomically
// check the registry and make a new instance only
// if one does not exist for its path, and otherwise
// return the existing instance.
func (r *badgerRegistrar) unprotectedRegister(w *BadgerDBWrapper) {
r.mp[w] = true
r.path2db[w.path] = w
}
// unregister removes w from r
func (r *badgerRegistrar) unregister(w *BadgerDBWrapper) {
r.mu.Lock()
delete(r.mp, w)
delete(r.path2db, w.path)
r.mu.Unlock()
}
func DumpAllBadger() {
globalBadgerReg.mu.Lock()
defer globalBadgerReg.mu.Unlock()
for w := range globalBadgerReg.mp {
_ = w
AlwaysPrintf("this badger path='%v' has: \n%v\n", w.path, w.StringifiedBadgerKeys(nil))
}
}
// newBadgerDBWrapper creates a new empty database, blowing away
// any prior path + "-badgerdb" directory.
func newBadgerDBWrapper(path string) (*BadgerDBWrapper, error) {
func (r *badgerRegistrar) newBadgerDBWrapper(path string) (*BadgerDBWrapper, error) {
bpath := badgerPath(path)
err := os.RemoveAll(bpath)
if err != nil {
return nil, err
}
return openBadgerDBWrapper(bpath)
return r.openBadgerDBWrapper(bpath)
}
// badgerPath is a helper for determining the full directory
@ -212,7 +270,12 @@ func badgerPath(path string) string {
// openBadgerDB opens the database in the bpath directoy
// without deleting any prior content. Any BadgerDB
// database directory will have the "-badgerdb" suffix.
func openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) {
//
// openBadgerDB will check the registry and make a new instance only
// if one does not exist for its bpath. Otherwise it returns
// the existing instance. This insures only one badgerDB
// per bpath in this pilosa node.
func (r *badgerRegistrar) openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) {
// now that newTxFactory can call us directly, we might not
// have the -badgerdb suffix.
@ -220,9 +283,33 @@ func openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) {
bpath += "-badgerdb"
}
r.mu.Lock()
defer r.mu.Unlock()
w, ok := r.path2db[bpath]
if ok {
// creates the effect of having only one badger open per pilosa node.
return w, nil
}
// otherwise, make a new badger and store it in globalBadgerReg
// regular: works on amd64, but 386 doesn't work.
opt := badger.DefaultOptions(bpath).WithLogger(badgerDefaultLogger)
opt.Compression = badgeroptions.None // turn off compression.
opt.ZSTDCompressionLevel = 0 // really, just in case.
// MaxCacheSize docs:
//
// how much data cache should hold in memory. A small size of
// cache means lower memory consumption and lookups/iterations
// would take longer. It is recommended to use a cache if you're
// using compression or encryption. If compression and
// encryption both are disabled, adding a cache will lead to
// unnecessary overhead which will affect the read performance.
// Setting size to zero disables the cache altogether.
opt.MaxCacheSize = 0
opt.LoadBloomsOnOpen = false // should speed up start-up time.
// to get memory only do:
//opt := badger.DefaultOptions("").WithLogger(badgerDefaultLogger).WithInMemory(true)
@ -231,12 +318,16 @@ func openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) {
return nil, err
}
halt := make(chan bool)
w := &BadgerDBWrapper{
w = &BadgerDBWrapper{
reg: r,
path: bpath,
db: db,
halt: halt,
hasher: NewBlake3Hasher(),
}
r.unprotectedRegister(w)
w.startStack = stack()
w.startBadgerGarbageCollectionBackgroundGoro()
return w, nil
}
@ -251,77 +342,8 @@ func (w *BadgerDBWrapper) DeleteIndex(indexName string) error {
if strings.Contains(indexName, "'") {
return fmt.Errorf("error: bad indexName `%v` in BadgerDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes.", indexName)
}
w.muDb.Lock()
defer w.muDb.Unlock()
// a) do key-ony iteration, no value fetch;
//
// b) do deletes in large batches, to avoid alot of txn overhead;
// per recommendation https://github.com/dgraph-io/badger/issues/598
//
// c) we do not, at present, try to maintain one large
// transaction with all the keys in a index in it. Because
// there can be too many keys. Hence the index will disappear
// in chucks of 100K keys, not atomically-all-at-once.
prefix := badgerIndexOnlyPrefix(indexName)
noMoreKeysWithPrefix := false
const maxDeletesPerTxn = 100000
for !noMoreKeysWithPrefix {
err := w.db.Update(func(txn *badger.Txn) error {
o := badger.DefaultIteratorOptions
o.AllVersions = false
o.PrefetchValues = false // key-only iteration, no values.
// note: panic: Unclosed iterator at time of Txn.Discard ? panic on segfault here?
// This means we messed up and Closed() the Database already; too early.
it := txn.NewIterator(o)
defer it.Close()
n := 0
goners := make([][]byte, 0, maxDeletesPerTxn)
for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
// KeyCopy() is required; Key() means corruption and possible segfault.
key := it.Item().KeyCopy(nil)
goners = append(goners, key)
n++
if n >= maxDeletesPerTxn {
break
}
}
if !it.ValidForPrefix(prefix) {
noMoreKeysWithPrefix = true // done with the full delete of up to maxDeletesPerTxn
}
for _, key := range goners {
if err := txn.Delete(key); err != nil {
return err
}
}
return nil // auto-commit happens
})
// err back from Update can be ErrConflict in case of
// a conflict. Badger docs: "Depending on the state
// of your application, you have the option to
// retry the operation if you receive this error."
panicOn(err)
} // end for: proceed to next bath of 100K keys
// Finally, run a garbage collection to delete values from the value log.
//
// "Only one GC is allowed at a time. If another value log GC
// is running, or DB has been closed, this would return an ErrRejected."
// -- https://godoc.org/github.com/dgraph-io/badger#DB.RunValueLogGC
// Still, we don't see a mutex inside the RunValueLogGC code, so
// lock muGC just to be sure.
w.muGC.Lock()
defer w.muGC.Unlock()
_ = w.db.RunValueLogGC(0.5)
return nil
return w.DeletePrefix(prefix)
}
// startBadgerGarbageCollectionBackgroundGoro handles Badger DB
@ -367,6 +389,9 @@ type BadgerDBWrapper struct {
path string
db *badger.DB
// track our registrar for Close / goro leak reporting purposes.
reg *badgerRegistrar
// openTx and openIt are BadgerDBWrapper scoped tables of all open
// transactions and iterators. These are primarily for debugging purposes.
// openTx and openIt should only be read/written after locking the muOpenTxIt mutex.
@ -404,6 +429,10 @@ type BadgerDBWrapper struct {
// safety because otherwise TestAPI_ImportColumnAttrs sees
// corrupted data.
doAllocZero bool
// stack() from our creation point, to track tests
// that haven't closed us.
startStack string
}
// unprotectedListOpenTxAsString is a debugging helper.
@ -437,24 +466,30 @@ func (w *BadgerDBWrapper) UnprotectedListOpenItAsString() (r string) {
// Read-only queries should set write to false, to allow more concurrency.
// Methods on a BadgerTx are thread-safe, and can be called from
// different goroutines.
func (w *BadgerDBWrapper) NewBadgerTx(write bool) (tx *BadgerTx) {
//
// initialIndexName is optional. It is set by the TxFactory from the Txo
// options provided at the Tx creation point. It allows us to recognize
// and isolate cross-index queries more quickly. It can always be empty ""
// but when set is highly useful for debugging. It has no impact
// on transaction behavior.
//
func (w *BadgerDBWrapper) NewBadgerTx(write bool, initialIndexName string) (tx *BadgerTx) {
w.muDb.Lock()
defer w.muDb.Unlock()
tx = &BadgerTx{
write: write,
tx: w.db.NewTransaction(write),
Db: w,
initloc: stack(),
doAllocZero: w.doAllocZero,
write: write,
tx: w.db.NewTransaction(write),
Db: w,
initloc: stack(),
doAllocZero: w.doAllocZero,
initialIndexName: initialIndexName,
}
//vv("NewBadgerTx(write=%v) top, p=%p", write, tx)
//pp("NewBadgerTx(write=%v) top, p=%p, stack=\n\n'%v'", write, tx, stack())
if w.openTx == nil {
w.openTx = make(map[*BadgerTx]bool)
}
//pp("NewBadgerTx(write=%v); p=%p; (currently open txn: '%v', its: '%v'). initloc:'%v'", write, tx, w.unprotectedListOpenTxAsString(), w.UnprotectedListOpenItAsString(), tx.initloc)
w.muOpenTxIt.Lock()
w.openTx[tx] = write
w.muOpenTxIt.Unlock()
@ -466,6 +501,7 @@ func (w *BadgerDBWrapper) Close() (err error) {
w.muDb.Lock()
defer w.muDb.Unlock()
if !w.closed {
w.reg.unregister(w)
close(w.halt)
w.closed = true
}
@ -501,8 +537,15 @@ type BadgerTx struct {
// for tracking txn boundary issues, track all the memory
// that we deploy for roaring containers, and zero it on
// transaction commit/rollback.
acMu sync.Mutex // protect ourAllocs and ourContainers
ourAllocs [][]byte
ourContainers []*roaring.Container
initialIndexName string
}
func (tx *BadgerTx) Type() string {
return BadgerTxn
}
func (tx *BadgerTx) UseRowCache() bool {
@ -521,6 +564,8 @@ func (tx *BadgerTx) UseRowCache() bool {
// to transaction commit.
func (tx *BadgerTx) overWriteOurAllocs() {
tx.acMu.Lock()
defer tx.acMu.Unlock()
for _, s := range tx.ourAllocs {
// The Go compiler recognizes the following pattern and inserts
@ -529,6 +574,10 @@ func (tx *BadgerTx) overWriteOurAllocs() {
// and https://codereview.appspot.com/137880043
for i := range s {
s[i] = 0
// or
// Seebs suggested we might see even more crashes :)
// but since it will be slow (no memclr), we'll leave the default 0 for now.
//s[i] = -2
}
}
// keep this around if we need to activate out-of-mmap memory access again.
@ -632,21 +681,57 @@ func badgerKey(index, field, view string, shard uint64, roaringContainerKey uint
prefix := badgerPrefix(index, field, view, shard)
ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
return append(prefix, ckey...)
bkey := append(prefix, ckey...)
MustValidateKey(bkey)
return bkey
}
var ckeyPartExpected = []byte(";ckey@")
// MustValidatekey will panic on a bad badgerKey with an informative message.
func MustValidateKey(bkey []byte) {
n := len(bkey)
if n < 56 {
panic(fmt.Sprintf("bkey too short min size is 56 but we see %v in '%v'", n, string(bkey)))
}
beforeCkey := bkey[n-26 : n-20]
if !bytes.Equal(beforeCkey, ckeyPartExpected) {
panic(fmt.Sprintf(`bkey did not have expected ";ckey@" at 26 bytes from the end of the bkey '%v'; instead had '%v'`, string(bkey), string(beforeCkey)))
}
}
func shardFromBadgerKey(bkey []byte) (shard uint64) {
MustValidateKey(bkey)
n := len(bkey)
// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615 -> idx:'i';fld:'f';vw:'standard';shd:'1
by := bkey[:n-27]
beg := bytes.LastIndex(by, []byte("'"))
if beg == -1 {
panic(fmt.Sprintf("bad bkey='%v' did not have single quote to being shard decoding", string(bkey)))
}
parseMe := string(by[beg+1:])
shard, err := strconv.ParseUint(parseMe, 10, 64)
if err != nil {
panic(fmt.Sprintf("could not parse parseMe '%v' in strconv.ParseUint(), error: '%v'", parseMe, err))
}
return shard
}
// badgerKeyAndPrefix returns the equivalent of badgerKey() and badgerPrefix() calls.
func badgerKeyAndPrefix(index, field, view string, shard uint64, roaringContainerKey uint64) (key, prefix []byte) {
prefix = badgerPrefix(index, field, view, shard)
ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
return append(prefix, ckey...), prefix
bkey := append(prefix, ckey...)
MustValidateKey(bkey)
return bkey, prefix
}
var _ = badgerKeyAndPrefix // keep linter happy
// badgerKeyExtractContainerKey extracts the containerKey from bkey.
func badgerKeyExtractContainerKey(bkey []byte) (containerKey uint64) {
MustValidateKey(bkey)
// The zero padding means that the container-key is always the last 20 bytes of the bkey.
//
// Be sure to catch the problematic case of a user passing in only a prefix. A prefix
@ -665,11 +750,15 @@ func badgerKeyExtractContainerKey(bkey []byte) (containerKey uint64) {
return
}
func badgerAllShardPrefix(index, field, view string) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:", index, field, view))
}
// badgerPrefix returns everything from badgerKey up to and
// including the '@' fune in a badger key. The prefix excludes the roaring container key itself.
// NB must be kept in sync with badgerKey() and badgerKeyExtractContainerKey().
func badgerPrefix(index, field, view string, shard uint64) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%x';ckey@", index, field, view, shard))
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@", index, field, view, shard))
}
// badgerIndexOnlyPrefix returns a prefix suitable for DeleteIndex and a key-scan to
@ -869,6 +958,38 @@ func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64
return exists, err
}
func (tx *BadgerTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
prefix := badgerAllShardPrefix(index, field, view)
bi := NewBadgerIterator(tx, prefix)
defer bi.Close()
bi.Seek(prefix)
if !bi.it.Valid() {
return
}
lastShard := uint64(0)
firstDone := false
for bi.Next() {
item := bi.it.Item()
key := item.Key()
shard := shardFromBadgerKey(key)
if firstDone {
if shard != lastShard {
sliceOfShards = append(sliceOfShards, shard)
}
lastShard = shard
} else {
// first time
lastShard = shard
firstDone = true
sliceOfShards = append(sliceOfShards, shard)
}
}
return
}
// key is the container key for the first roaring Container
// roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will
// return the first container at or after key. found will be true if a
@ -877,10 +998,10 @@ func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64
// BadgerTx notes: We auto-stop at the end of this shard, not going beyond.
func (tx *BadgerTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
// needle example: "index:'i';field:'f';view:'v';shard:'0';key@00000000000000000000"
// needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000"
needle := badgerKey(index, field, view, shard, firstRoaringContainerKey)
// prefix example: "index:'i';field:'f';view:'v';shard:'0';key@"
// prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@"
prefix := badgerPrefix(index, field, view, shard)
bi := NewBadgerIterator(tx, prefix)
@ -917,12 +1038,6 @@ func NewBadgerIterator(tx *BadgerTx, prefix []byte) (bi *BadgerIterator) {
tx.Db.muOpenTxIt.Lock()
defer tx.Db.muOpenTxIt.Unlock()
defer func() {
r := recover()
if r != nil {
panic(r)
}
}()
opts := badger.DefaultIteratorOptions
opts.PrefetchValues = false // else by default, pre-fetches the 1st 100 values, which would be slow.
opts.Reverse = false
@ -1125,6 +1240,7 @@ func (tx *BadgerTx) Count(index, field, view string, shard uint64) (uint64, erro
}
// Max is the maximum bit-value in your bitmap.
// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does.
func (tx *BadgerTx) Max(index, field, view string, shard uint64) (uint64, error) {
prefix := badgerPrefix(index, field, view, shard)
@ -1133,7 +1249,10 @@ func (tx *BadgerTx) Max(index, field, view string, shard uint64) (uint64, error)
it := NewBadgerReverseIterator(tx, prefix, seekto) // this iterator is still open, when we commit/discard tx.
defer it.Close()
hb, rc := it.Value()
if !it.it.Valid() {
return 0, nil
}
hb, rc := it.Value() // getting it returns invalid, as in empty iterator
lb := rc.Max()
return hb<<16 | uint64(lb), nil
@ -1309,7 +1428,7 @@ func (tx *BadgerTx) IncrementOpN(index, field, view string, shard uint64, change
// ImportRoaringBits handles deletes by setting clear=true.
// rowSet[rowID] returns the number of bit changed on that rowID.
func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
n := itr.Len()
if n == 0 {
return
@ -1456,11 +1575,19 @@ func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
// TODO: performance tuning might want w := v here, if we can guarantee no access to memory past the Tx lifetime.
//
// Problem is, at least some tests appear to not respect transaction boundaries...
//
// Seebs suggested this nice variation: we could use individual mmaps for these
// copies, which would be unusable in production, but workable for testing, and then unmap them,
// which would get us probable segfaults on future accesses to them.
//
w := make([]byte, len(v))
copy(w, v) // green go test -v -run TestAPI_ImportColumnAttrs
copy(w, v)
// the copy above makes green: // green go test -v -run TestAPI_ImportColumnAttrs
//w := v // if instead of append we use v directly, it causes red: go test -v -run TestAPI_ImportColumnAttrs
// register w so we can catch out-of-tx memory access
tx.acMu.Lock()
defer tx.acMu.Unlock()
tx.ourAllocs = append(tx.ourAllocs, w)
switch typ {
@ -1511,7 +1638,7 @@ func fromInterval16(a []roaring.Interval16) []byte {
// keys available in badger.
func (w *BadgerDBWrapper) StringifiedBadgerKeys(optionalUseThisTx Tx) (r string) {
if optionalUseThisTx == nil {
tx := w.NewBadgerTx(!writable)
tx := w.NewBadgerTx(!writable, "<StringifiedBadgerKeys>")
defer tx.Rollback()
r = stringifiedBadgerKeysTx(tx)
return
@ -1699,3 +1826,94 @@ func dirAsString(path string) (r string) {
}
var _ = dirAsString // happy linter
func (w *BadgerDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
prefix := badgerPrefix(index, field, view, shard)
return w.DeletePrefix(prefix)
}
func (w *BadgerDBWrapper) DeletePrefix(prefix []byte) error {
w.muDb.Lock()
defer w.muDb.Unlock()
// a) do key-ony iteration, no value fetch;
//
// b) do deletes in large batches, to avoid alot of txn overhead;
// per recommendation https://github.com/dgraph-io/badger/issues/598
//
// c) we do not, at present, try to maintain one large
// transaction with all the keys in a index in it. Because
// there can be too many keys. Hence the index will disappear
// in chucks of 100K keys, not atomically-all-at-once.
noMoreKeysWithPrefix := false
const maxDeletesPerTxn = 100000
for !noMoreKeysWithPrefix {
err := w.db.Update(func(txn *badger.Txn) error {
o := badger.DefaultIteratorOptions
o.AllVersions = false
o.PrefetchValues = false // key-only iteration, no values.
// note: panic: Unclosed iterator at time of Txn.Discard ? panic on segfault here?
// This means we messed up and Closed() the Database already; too early.
it := txn.NewIterator(o)
defer it.Close()
n := 0
goners := make([][]byte, 0, maxDeletesPerTxn)
for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
// KeyCopy() is required; Key() means corruption and possible segfault.
key := it.Item().KeyCopy(nil)
goners = append(goners, key)
n++
if n >= maxDeletesPerTxn {
break
}
}
if !it.ValidForPrefix(prefix) {
noMoreKeysWithPrefix = true // done with the full delete of up to maxDeletesPerTxn
}
for _, key := range goners {
if err := txn.Delete(key); err != nil {
return err
}
}
return nil // auto-commit happens
})
// err back from Update can be ErrConflict in case of
// a conflict. Badger docs: "Depending on the state
// of your application, you have the option to
// retry the operation if you receive this error."
panicOn(err)
} // end for: proceed to next bath of 100K keys
// Finally, run a garbage collection to delete values from the value log.
//
// "Only one GC is allowed at a time. If another value log GC
// is running, or DB has been closed, this would return an ErrRejected."
// -- https://godoc.org/github.com/dgraph-io/badger#DB.RunValueLogGC
// Still, we don't see a mutex inside the RunValueLogGC code, so
// lock muGC just to be sure.
w.muGC.Lock()
defer w.muGC.Unlock()
_ = w.db.RunValueLogGC(0.5)
return nil
}
func (tx *BadgerTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
rbm, err := tx.RoaringBitmap(index, field, view, shard)
if err != nil {
return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap")
}
var buf bytes.Buffer
sz, err = rbm.WriteTo(&buf)
if err != nil {
return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)")
}
return ioutil.NopCloser(&buf), sz, err
}

263
badger_test.go
View file

@ -12,12 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// explanation of build tags:
//
// badgerdb builds but won't run in 32-bit 386 world, as of 2020 July 20.
// See https://github.com/dgraph-io/badger/issues/1384 for any progress.
// What we see is that the value-log allocations immediately run out of
// memory. So we turn off 386 with a build tag to keep the .circleci happy.
//
// gendebug_test will have a TestMain if build tag generationdebug is on,
// so we avoid conflicting with that debug scenario.
// +build !386
// +build !generationdebug
package pilosa
@ -40,7 +46,7 @@ var _ = &roaring.Bitmap{}
func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) {
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
@ -53,7 +59,7 @@ func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string
func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) {
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
@ -64,7 +70,7 @@ func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view str
}
func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
// add a bit
changed, err := tx.Add(index, field, view, shard, doBatched, putme)
@ -82,14 +88,14 @@ func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string,
}
func badgerDBMustDeleteBitvalueContainer(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
hi := highbits(putme)
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
panicOn(tx.Commit())
}
func badgerDBMustDeleteBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
_, err := tx.Remove(index, field, view, shard, putme)
panicOn(err)
panicOn(tx.Commit())
@ -99,7 +105,7 @@ func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func()
var err error
fn := badgerPath(path)
panicOn(os.RemoveAll(fn))
w, err = newBadgerDBWrapper(path)
w, err = globalBadgerReg.newBadgerDBWrapper(path)
panicOn(err)
// verify it is empty
@ -109,6 +115,7 @@ func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func()
}
return w, func() {
w.Close() // stop any started background GC goroutine.
os.RemoveAll(fn)
}
}
@ -118,8 +125,8 @@ func TestBadger_SetBitmap(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SetBitmap")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
bitvalue := uint64(0)
changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
if changed <= 0 {
@ -140,7 +147,7 @@ func TestBadger_SetBitmap(t *testing.T) {
// commited, so should be visible outside the txn
//
tx2 := dbwrap.NewBadgerTx(!writable)
tx2 := dbwrap.NewBadgerTx(!writable, index)
exists, err = tx2.Contains(index, field, view, shard, bitvalue)
panicOn(err)
if !exists {
@ -159,9 +166,9 @@ func TestBadger_OffsetRange(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SetBitmap")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
bitvalue := uint64(1 << 20)
changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
if changed <= 0 {
@ -194,7 +201,7 @@ func TestBadger_OffsetRange(t *testing.T) {
start := uint64(0 << 16)
endx := bitvalue + 1<<16
tx2 := dbwrap.NewBadgerTx(!writable)
tx2 := dbwrap.NewBadgerTx(!writable, index)
rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
tx2.Rollback()
@ -208,7 +215,7 @@ func TestBadger_OffsetRange(t *testing.T) {
// now offset by 2M
offset = uint64(2 << 20)
tx3 := dbwrap.NewBadgerTx(!writable)
tx3 := dbwrap.NewBadgerTx(!writable, index)
rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
tx3.Rollback()
@ -236,7 +243,7 @@ func TestBadger_Count_on_many_containers(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
n, err := tx.Count(index, field, view, shard)
@ -252,7 +259,7 @@ func TestBadger_Count_dense_containers(t *testing.T) {
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
expected := 0
// can't do more than about 100k writes per badger txn by default, so
@ -280,9 +287,9 @@ func TestBadger_ContainerIterator_on_empty(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(!writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
bitvalue := uint64(0)
citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue)
panicOn(err)
@ -298,9 +305,9 @@ func TestBadger_ContainerIterator_on_one_bit(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
bitvalue := uint64(42)
@ -398,9 +405,9 @@ func TestBadger_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
putme := uint64(1<<16) + 3 // in the key:1 container
searchme := putme + 1
@ -453,9 +460,9 @@ func TestBadger_ContainerIterator_empty_iteration_loop(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_empty_iteration_loop")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
putme := uint64(1<<16) + 3 // in the key:1 container
searchme := uint64(1 << 17) // in the next container, key:2
@ -503,9 +510,9 @@ func TestBadger_ForEach_on_one_bit(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
bitvalue := uint64(42)
@ -563,7 +570,7 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// delete, but rollback instead of commit
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
hi := highbits(putme)
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
tx.Rollback()
@ -572,7 +579,7 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// c) within one Tx, after delete it should be gone as viewed within the txn.
tx = dbwrap.NewBadgerTx(writable)
tx = dbwrap.NewBadgerTx(writable, index)
hi = highbits(putme)
exists, err := tx.Contains(index, field, view, shard, putme)
@ -624,7 +631,7 @@ func TestBadger_Remove_one_bit_test(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// delete, but rollback instead of commit
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
hi, lo := highbits(putme), lowbits(putme)
_, _ = hi, lo
_, err := tx.Remove(index, field, view, shard, hi)
@ -635,7 +642,7 @@ func TestBadger_Remove_one_bit_test(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// c) within one Tx, after delete it should be gone as viewed within the txn.
tx = dbwrap.NewBadgerTx(writable)
tx = dbwrap.NewBadgerTx(writable, index)
exists, err := tx.Contains(index, field, view, shard, putme)
panicOn(err)
@ -724,7 +731,7 @@ func TestBadger_Max_on_many_containers(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
max, err := tx.Max(index, field, view, shard)
@ -742,7 +749,7 @@ func TestBadger_Min_on_many_containers(t *testing.T) {
index, field, view, shard := "i", "f", "v", uint64(0)
// verify no containers flag works
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
min, containersExist, err := tx.Min(index, field, view, shard)
_ = min
panicOn(err)
@ -759,7 +766,7 @@ func TestBadger_Min_on_many_containers(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx = dbwrap.NewBadgerTx(!writable)
tx = dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
min, containersExist, err = tx.Min(index, field, view, shard)
@ -780,7 +787,7 @@ func TestBadger_CountRange_on_many_containers(t *testing.T) {
index, field, view, shard := "i", "f", "v", uint64(0)
// verify no containers flag works
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
panicOn(err)
if n != 0 {
@ -796,7 +803,7 @@ func TestBadger_CountRange_on_many_containers(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx = dbwrap.NewBadgerTx(!writable)
tx = dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
@ -824,7 +831,7 @@ func TestBadger_CountRange_middle_container(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
// pick out just the middle container with the 1 bit set on it.
@ -849,7 +856,7 @@ func TestBadger_CountRange_many_middle_container(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
// get them all
@ -880,7 +887,7 @@ func TestBadger_UnionInPlace(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx2 := dbwrap.NewBadgerTx(!writable)
tx2 := dbwrap.NewBadgerTx(!writable, index)
n, err := tx2.Count(index, field, view, shard)
panicOn(err)
if n != 2 {
@ -895,7 +902,7 @@ func TestBadger_UnionInPlace(t *testing.T) {
}
mustAddR(others3.Add(4 << 16)) // outside the 2<<16 container
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
err = tx.UnionInPlace(index, field, view, shard, others, others2, others3)
panicOn(err)
@ -920,7 +927,7 @@ func TestBadger_RoaringBitmap(t *testing.T) {
putme := expected
badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
rbm, err := tx.RoaringBitmap(index, field, view, shard)
@ -951,10 +958,8 @@ func TestBadger_reverse_badger_iterator_and_prefix_valid(t *testing.T) {
return nil
})
panicOn(err)
//vv("stringifiedBadgerKeys(db) = '%v'", stringifiedBadgerKeys(dbwrap.db))
// allkeys:["a:0", "a:1", "a:2", "b:0", "b:1", "b:2", "c:0", "c:1", "c:2", ]'
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, "no-index-avail")
prefix := []byte("b:")
it := NewBadgerIterator(tx, prefix)
@ -1014,10 +1019,8 @@ func TestBadger_just_reverse_badger_iterator_and_prefix_valid(t *testing.T) {
return nil
})
panicOn(err)
//vv("stringifiedBadgerKeys(db) = '%v'", stringifiedBadgerKeys(dbwrap.db))
// allkeys:["a:0", "a:1", "a:2", "b:0", "b:1", "b:2", "c:0", "c:1", "c:2", ]'
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, "no-index-avail")
seekto := []byte("c:")
prefix := []byte("b:")
@ -1046,9 +1049,9 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
//bitvalue := uint64(42)
@ -1062,7 +1065,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
clear := false
logme := false
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
_ = rowSet
if changed != len(bits) {
panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err))
@ -1079,7 +1082,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
// now test the union in place with the same set gives no change.
changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
_ = rowSet
if changed != 0 {
panic(fmt.Sprintf("should have not changed any bits on the second import, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err))
@ -1103,7 +1106,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
itr, err := roaring.NewRoaringIterator(data)
panicOn(err)
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
_ = rowSet
if changed != 1 {
panic(fmt.Sprintf("should have changed 1 bit: '%v', rowSet='%#v', err='%v'", changed, rowSet, err))
@ -1128,9 +1131,9 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits_set_nonoverlapping_bits")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
// get some roaring bits, get an itr RoaringIterator from them
rowSize := uint64(0)
@ -1148,7 +1151,7 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
clear := false
logme := false
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
_ = rowSet
if changed != len(bits) {
panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err))
@ -1165,7 +1168,7 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
// now import the 2nd, overlapping set and set them.
changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize)
changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil)
_ = rowSet
if changed != 4 {
panic(fmt.Sprintf("should have changed 2 bits: the 1 and the 3, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err))
@ -1178,9 +1181,9 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits_clear_nonoverlapping_bits")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
// get some roaring bits, get an itr RoaringIterator from them
rowSize := uint64(0)
@ -1198,7 +1201,7 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
clear := false
logme := false
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
_ = rowSet
if changed != len(bits) {
panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err))
@ -1216,7 +1219,7 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
// now import the 2nd overlapping set and clear them.
clear = true
changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize)
changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil)
_ = rowSet
if changed != 2 {
panic(fmt.Sprintf("should have changed 1 bit: the 2, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err))
@ -1252,9 +1255,9 @@ func TestBadger_DeleteIndex(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
bitvalue := uint64(777)
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
bitvalue := uint64(777)
bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16}
for _, v := range bits {
changed, err := tx.Add(index, field, view, shard, doBatched, v)
@ -1290,7 +1293,7 @@ func TestBadger_DeleteIndex(t *testing.T) {
err = dbwrap.DeleteIndex(index)
panicOn(err)
tx = dbwrap.NewBadgerTx(!writable)
tx = dbwrap.NewBadgerTx(!writable, index2)
defer tx.Rollback()
exists, err = tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
@ -1314,9 +1317,9 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex_over100k")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
bitvalue := uint64(777)
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
bitvalue := uint64(777)
limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction.
//limit := uint64(101)
for v := uint64(1); v < limit; v++ {
@ -1328,7 +1331,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) {
panicOn(err)
if v%100000 == 0 {
panicOn(tx.Commit())
tx = dbwrap.NewBadgerTx(writable)
tx = dbwrap.NewBadgerTx(writable, index)
}
}
@ -1345,7 +1348,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) {
err = dbwrap.DeleteIndex(index)
panicOn(err)
tx = dbwrap.NewBadgerTx(!writable)
tx = dbwrap.NewBadgerTx(!writable, index2)
defer tx.Rollback()
exists, err := tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
@ -1445,3 +1448,135 @@ func mustAddR(changed bool, err error) {
func mustRemove(changeCount int, err error) {
panicOn(err)
}
func TestBadger_DeleteFragment(t *testing.T) {
// setup
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteFragment")
defer clean()
defer dbwrap.Close()
index, field, view, shard0 := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
shard1 := uint64(1)
bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16}
shards := []uint64{shard0, shard1}
for _, s := range shards {
for _, v := range bits {
changed, err := tx.Add(index, field, view, s, doBatched, v)
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
}
}
for _, s := range shards {
for _, v := range bits {
exists, err := tx.Contains(index, field, view, s, v)
panicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!!")
}
}
}
err := tx.Commit()
panicOn(err)
//vv("Dump: %v", dbwrap.StringifiedBadgerKeys(nil))
// end of setup
survivor := shard0
victim := shard1
err = dbwrap.DeleteFragment(index, field, view, victim, nil)
panicOn(err)
tx = dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
for _, s := range shards {
for _, v := range bits {
exists, err := tx.Contains(index, field, view, s, v)
panicOn(err)
if s == survivor {
if !exists {
panic(fmt.Sprintf("ARG survivor died : bit %v", v))
}
} else if s == victim { // victim, should have been deleted
if exists {
panic(fmt.Sprintf("ARG victim lived : bit %v", v))
}
}
}
}
}
func TestBadger_shardFromBadgerKey(t *testing.T) {
if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615")) != 1 {
panic("problem")
}
if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'0';ckey@18446744073709551615")) != 0 {
panic("problem")
}
if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@18446744073709551615")) != 18446744073709551615 {
panic("problem")
}
func() {
defer func() {
r := recover()
if r == nil {
panic("should have panic-ed")
}
}()
// called for the panic of a short ckey, only 19 bytes instead of 20
shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@1844674407370955161"))
}()
}
func TestBadger_SliceOfShards(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SliceOfShards")
defer clean()
defer dbwrap.Close()
index, field, view := "i", "f", "v"
shards := []uint64{0, 1, 2, 3, 1000001, 2000001}
putme := uint64(179)
for _, shard := range shards {
badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
slc, err := tx.SliceOfShards(index, field, view, "")
panicOn(err)
for i := range shards {
if shards[i] != slc[i] {
panic(fmt.Sprintf("expected at i=%v that slc[i]=%v = shards[i]=%v", i, slc[i], shards[i]))
}
}
}
func reportTestBadgersNeedingClose() {
globalBadgerReg.mu.Lock()
defer globalBadgerReg.mu.Unlock()
n := len(globalBadgerReg.mp)
if n > 0 {
AlwaysPrintf("*** these badgers are still open (n=%v):", n)
i := 0
for w := range globalBadgerReg.mp {
AlwaysPrintf("i=%v, w p=%p stack:\n%v\n\n", i, w, w.startStack)
i++
}
}
}
var _ = reportTestBadgersNeedingClose // happy linter
func TestMain(m *testing.M) {
ret := m.Run()
//reportTestBadgersNeedingClose()
os.Exit(ret)
}

284
bluegreentx.go
View file

@ -16,6 +16,9 @@ package pilosa
import (
"fmt"
"io"
"reflect"
"sort"
"github.com/pilosa/pilosa/v2/roaring"
)
@ -27,6 +30,8 @@ type blueGreenTx struct {
b Tx // b's output is returned
idx *Index
checker blueGreenChecker
}
func newBlueGreenTx(a, b Tx, idx *Index) *blueGreenTx {
@ -37,6 +42,10 @@ var _ = newBlueGreenTx // keep linter happy
var _ Tx = (*blueGreenTx)(nil)
func (c *blueGreenTx) Type() string {
return c.a.Type() + "_" + c.b.Type()
}
func (c *blueGreenTx) Readonly() bool {
a := c.a.Readonly()
b := c.b.Readonly()
@ -47,6 +56,7 @@ func (c *blueGreenTx) Readonly() bool {
}
func (c *blueGreenTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
c.checker.see(index, field, view, shard)
return c.b.NewTxIterator(index, field, view, shard)
}
@ -55,11 +65,71 @@ func (c *blueGreenTx) Pointer() string {
}
func (c *blueGreenTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
c.checker.see(index, field, view, shard)
c.a.IncrementOpN(index, field, view, shard, changedN)
c.b.IncrementOpN(index, field, view, shard, changedN)
}
func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
here := fmt.Sprintf("%v/%v/%v/%v", index, field, view, shard)
aIter, aFound, aErr := c.a.ContainerIterator(index, field, view, shard, 0)
bIter, bFound, bErr := c.b.ContainerIterator(index, field, view, shard, 0)
if aFound != bFound {
panic(fmt.Sprintf("compareTxState[%v]: A ContainerIterator had aFound=%v, but B had bFound=%v; at '%v'", here, aFound, bFound, stack()))
}
if aErr == nil {
defer aIter.Close()
}
if bErr == nil {
defer bIter.Close()
}
if aErr != nil || bErr != nil {
if aErr != nil && bErr != nil {
panic(fmt.Sprintf("compareTxState[%v]: A reported err '%v'; B reported err '%v' at %v", here, aErr, bErr, stack()))
}
if aErr != nil {
panic(fmt.Sprintf("compareTxState[%v]: A reported err %v at %v; but B did not", here, aErr, stack()))
}
if bErr != nil {
panic(fmt.Sprintf("compareTxState[%v]: B reported err %v at %v; but A did not", here, bErr, stack()))
}
}
for aIter.Next() {
aKey, aValue := aIter.Value()
if !bIter.Next() {
panic(fmt.Sprintf("compareTxState[%v]: A found key %v, B didn't, at %v", here, aKey, stack()))
}
bKey, bValue := bIter.Value()
if bKey != aKey {
panic(fmt.Sprintf("compareTxState[%v]: A found key %v, B found %v, at %v", here, aKey, bKey, stack()))
}
if err := aValue.BitwiseCompare(bValue); err != nil {
panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v at %v", here, aKey, err, stack()))
}
}
// end checking everything in A, but does B have more?
if bIter.Next() {
bKey, _ := bIter.Value()
panic(fmt.Sprintf("compareTxState[%v]: B found key %v, A didn't, at %v", here, bKey, stack()))
}
}
func (c *blueGreenTx) checkDatabase() {
for index, fields := range c.checker.seen() {
for field, views := range fields {
for view, shards := range views {
for shard := range shards {
c.compareTxState(index, field, view, shard)
}
}
}
}
}
func (c *blueGreenTx) Rollback() {
c.checkDatabase()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
@ -71,6 +141,7 @@ func (c *blueGreenTx) Rollback() {
}
func (c *blueGreenTx) Commit() error {
c.checkDatabase()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
@ -86,6 +157,7 @@ func (c *blueGreenTx) Commit() error {
}
func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
@ -100,6 +172,7 @@ func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*r
}
func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
@ -116,6 +189,7 @@ func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uin
}
func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
@ -126,17 +200,11 @@ func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key
errB := c.b.PutContainer(index, field, view, shard, key, rc)
compareErrors(errA, errB)
/* draft idea of how to check the full databases afterwards:
hashA := c.a.RootHashString()
hashB := c.b.RootHashString()
if hashA != hashB {
panic(fmt.Sprintf("hashA = '%v' but hashB = '%v'", hashA, hashB))
}
*/
return errB
}
func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
@ -147,49 +215,37 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64,
// remember where the iterator started, so we can replay it a second time.
rit2 := rit.Clone()
changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize)
changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize, data)
if changedA != changedB {
panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB))
}
if len(rowSetA) != len(rowSetB) {
panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB))
}
for k, va := range rowSetA {
vb, ok := rowSetB[k]
if !ok {
panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB))
if len(data) == 0 {
// okay to check! otherwise we are in the fragment.fillFragmentFromArchive
// case where we know that RoaringTx.ImportRoaringBits changed and rowSet will
// be inaccurate.
if changedA != changedB {
panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB))
}
if va != vb {
panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb))
if len(rowSetA) != len(rowSetB) {
panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB))
}
for k, va := range rowSetA {
vb, ok := rowSetB[k]
if !ok {
panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB))
}
if va != vb {
panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb))
}
}
}
compareErrors(errA, errB)
//compareDatabases(c.a, c.b)
return changedB, rowSetB, errB
}
/* // TODO: get a database-wide checksum working
func compareDatabases(a, b Tx) {
index, field, view, shard := "i", "f", "v", uint64(0)
ha, errA := a.WholeDatabaseBlake3Hash(index, field, view, shard)
panicOn(errA)
hb, errB := b.WholeDatabaseBlake3Hash(index, field, view, shard)
panicOn(errB)
if ha != hb {
panic(fmt.Sprintf("a.WholeDatabaseBlake3Hash(%T) = '%v' but b.WholeDatabaseBlake3Hash(%T) = '%v'", a, ha, b, hb))
}
}
*/
func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
@ -207,6 +263,7 @@ func (c *blueGreenTx) UseRowCache() bool {
}
func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Add() panic '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, stack())
@ -249,6 +306,7 @@ func compareErrors(errA, errB error) {
}
func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
@ -263,6 +321,7 @@ func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint6
}
func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
@ -278,6 +337,7 @@ func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint
}
func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
@ -290,10 +350,14 @@ func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64,
bit, bfound, errB := c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
compareErrors(errA, errB)
if errA != nil {
ait.Close() // don't leak it.
}
return bit, bfound, errB
}
func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
@ -310,6 +374,7 @@ func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i
}
func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
@ -326,6 +391,7 @@ func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, star
}
func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
@ -342,6 +408,7 @@ func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, er
}
func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
@ -358,6 +425,7 @@ func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, erro
}
func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
@ -374,6 +442,7 @@ func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool
}
func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
@ -387,6 +456,7 @@ func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, othe
}
func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
@ -405,6 +475,7 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start,
}
func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
@ -419,3 +490,136 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star
compareErrors(errA, errB)
return b, errB
}
func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
rcA, szA, errA := c.a.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
rcB, szB, errB := c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
if szA != szB {
panic(fmt.Sprintf("szA = %v, but szB = %v", szA, szB))
}
compareErrors(errA, errB)
return &MultiReaderB{a: rcA, b: rcB}, szB, errB
}
func (c *blueGreenTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
// doesn't change state, so we don't really need see() call here. And we don't have a single shard for it.
//c.checker.see(index, field, view, shard) // don't have shard.
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
slcA, errA := c.a.SliceOfShards(index, field, view, optionalViewPath)
slcB, errB := c.b.SliceOfShards(index, field, view, optionalViewPath)
compareErrors(errA, errB)
// sort order may be different, and that's ok.
cpa := append([]uint64{}, slcA...)
cpb := append([]uint64{}, slcB...)
sort.Slice(cpa, func(i, j int) bool { return cpa[i] < cpa[j] })
sort.Slice(cpb, func(i, j int) bool { return cpb[i] < cpb[j] })
if !reflect.DeepEqual(cpa, cpb) {
// report the first difference
ma := make(map[uint64]bool)
for _, ka := range slcA {
ma[ka] = true
}
for _, kb := range slcB {
if !ma[kb] {
panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B had %v, but A did not; in the SliceOfShards returned slice.", kb))
}
delete(ma, kb)
}
if len(ma) != 0 {
for _, firstDifference := range ma {
panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! A had %v, but B did not; in the SliceOfShards returned slice.", firstDifference))
}
}
panic(fmt.Sprintf("blueGreenTx SliceOfShards diference \n slcA='%#v';\n slcB='%#v';\n", cpa, cpb))
}
return slcB, errB
}
type MultiReaderB struct {
a io.ReadCloser
b io.ReadCloser
}
// TODO(jea): test this for accuracy/correctness.
func (m *MultiReaderB) Read(p []byte) (nB int, errB error) {
nB, errB = m.b.Read(p)
p2 := make([]byte, nB)
// discard the exact same amount from A
// ReadAtLeast reads from r into buf until it has read at least
// min bytes. It returns the number of bytes copied and an error
// if fewer bytes were read. The error is EOF only if no bytes
// were read. If an EOF happens after reading fewer than min bytes,
// ReadAtLeast returns ErrUnexpectedEOF. If min is greater than
// the length of buf, ReadAtLeast returns ErrShortBuffer. On
// return, n >= min if and only if err == nil. If r returns
// an error having read at least min bytes, the error is dropped.
nA, errA := io.ReadAtLeast(m.a, p2, nB)
if errA == io.ErrUnexpectedEOF {
panic(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA))
}
if nA != nB {
panic(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA))
}
return
}
func (m *MultiReaderB) Close() error {
m.a.Close()
return m.b.Close()
}
// blueGreenChecker is used
type blueGreenChecker struct {
visited map[string]map[string]map[string]map[uint64]struct{}
done bool
}
// see would mark a thing as seen.
func (b *blueGreenChecker) see(index, field, view string, shard uint64) {
if b.visited == nil {
b.visited = make(map[string]map[string]map[string]map[uint64]struct{})
}
var visitedIdx map[string]map[string]map[uint64]struct{}
var visitedField map[string]map[uint64]struct{}
var visitedView map[uint64]struct{}
if visitedIdx = b.visited[index]; visitedIdx == nil {
visitedIdx = make(map[string]map[string]map[uint64]struct{})
b.visited[index] = visitedIdx
}
if visitedField = visitedIdx[field]; visitedField == nil {
visitedField = make(map[string]map[uint64]struct{})
visitedIdx[field] = visitedField
}
if visitedView = visitedField[view]; visitedView == nil {
visitedView = make(map[uint64]struct{})
visitedField[view] = visitedView
}
visitedView[shard] = struct{}{}
}
// seen reports the things it has seen, exactly once so
// that Rollback can be called after Commit without repeating
// the check.
func (b *blueGreenChecker) seen() map[string]map[string]map[string]map[uint64]struct{} {
if b.done {
return nil
}
b.done = true
return b.visited
}

28
catcher.go
View file

@ -16,6 +16,7 @@ package pilosa
import (
"fmt"
"io"
"github.com/pilosa/pilosa/v2/roaring"
)
@ -51,14 +52,14 @@ func (c *catcherTx) WholeDatabaseBlake3Hash(index, field, view string, shard uin
return c.b.WholeDatabaseBlake3Hash(index, field, view, shard)
}
func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
}
func (c *catcherTx) Readonly() bool {
@ -275,3 +276,26 @@ func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start,
}()
return c.b.OffsetRange(index, field, view, shard, offset, start, end)
}
func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
}
func (c *catcherTx) Type() string {
return c.b.Type()
}
func (c *catcherTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.SliceOfShards(index, field, view, optionalViewPath)
}

2
cluster.go
View file

@ -525,7 +525,7 @@ func (c *cluster) unprotectedSetState(state string) {
cleaner.Cluster = c
cleaner.Closing = c.closing
// Clean holder.
// Clean holder. This is where the shard gets removed after resize.
if err := cleaner.CleanHolder(); err != nil {
c.logger.Printf("holder clean error: err=%s", err)
}

13
cluster_internal_test.go
View file

@ -96,7 +96,9 @@ func newIndexWithTempPath(name string) *Index {
if err != nil {
panic(err)
}
index, err := NewIndex(NewHolder(DefaultPartitionN), path, name)
h := NewHolder(DefaultPartitionN)
h.Path = path
index, err := h.CreateIndex(name, IndexOptions{})
if err != nil {
panic(err)
}
@ -160,7 +162,7 @@ func TestFragSources(t *testing.T) {
defer idx.Close()
// Obtain transaction.
tx := &RoaringTx{Index: idx}
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx})
defer tx.Rollback()
field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
@ -809,12 +811,12 @@ func TestCluster_ResizeStates(t *testing.T) {
if idx0 == nil {
t.Fatal(`idx0 was nil, could not retrieve Index("i")`)
}
//idx0.Dump("node0")
// addNode needs to block until the resize process has completed.
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
@ -846,8 +848,6 @@ func TestCluster_ResizeStates(t *testing.T) {
if idx1 == nil {
t.Fatal(`idx1 was nil, could not retrieve Index("i")`)
}
//idx0.Dump("after rebalance, node0")
//idx1.Dump("after rebalance, node1")
// Ensure checksums are the same.
if chksum, err := node1Fragment.Checksum(); err != nil {
@ -872,6 +872,7 @@ func TestAE(t *testing.T) {
c.abortAntiEntropy()
close(ch)
}()
defer c.abortAntiEntropyQ() // avoid leaking a goroutine.
select {
case <-ch:
return
@ -883,11 +884,13 @@ func TestAE(t *testing.T) {
t.Run("AbortBlocksInitialized", func(t *testing.T) {
c := newCluster()
c.initializeAntiEntropy()
ch := make(chan struct{})
go func() {
c.abortAntiEntropy()
close(ch)
}()
defer c.abortAntiEntropyQ() // avoid leak of goroutine.
select {
case <-ch:
t.Fatalf("aborting anti entropy on an initialized cluster didn't block")

282
executor.go
View file

@ -19,12 +19,12 @@ import (
"encoding/json"
"fmt"
"math"
"math/bits"
"sort"
"strings"
"sync"
"time"
"github.com/molecula/ext"
"github.com/pilosa/pilosa/v2/pql"
pb "github.com/pilosa/pilosa/v2/proto"
"github.com/pilosa/pilosa/v2/roaring"
@ -64,12 +64,6 @@ type executor struct {
workersWG sync.WaitGroup
workerPoolSize int
work chan job
// global registry to check for name clashes
additionalOps map[string]*ext.BitmapOp
// typed registries we can use in lookups
additionalBitmapOps map[string]ext.BitmapOpBitmap
additionalCountOps map[string]ext.BitmapOpUnaryCount
additionalFieldOps map[string]ext.BitmapOpBSIBitmap
}
// executorOption is a functional option type for pilosa.Executor
@ -125,38 +119,6 @@ func (e *executor) Close() error {
return nil
}
func (e *executor) registerOps(ops []ext.BitmapOp) error {
if e.additionalOps == nil {
e.additionalOps = make(map[string]*ext.BitmapOp)
e.additionalBitmapOps = make(map[string]ext.BitmapOpBitmap)
e.additionalCountOps = make(map[string]ext.BitmapOpUnaryCount)
e.additionalFieldOps = make(map[string]ext.BitmapOpBSIBitmap)
}
for i, op := range ops {
name := op.Name
if _, exists := e.additionalOps[name]; exists {
return fmt.Errorf("op name '%s' already defined", name)
}
e.additionalOps[name] = &ops[i]
typ := ops[i].Func.BitmapOpType()
switch {
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
e.additionalCountOps[name] = ops[i].Func.(ext.BitmapOpUnaryCount)
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
e.additionalBitmapOps[name] = ops[i].Func.(ext.BitmapOpBitmap)
case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
if fn, ok := ops[i].Func.(ext.BitmapOpBSIBitmapPrecall); ok {
e.additionalFieldOps[name] = ext.BitmapOpBSIBitmap(fn)
} else {
e.additionalFieldOps[name] = ops[i].Func.(ext.BitmapOpBSIBitmap)
}
default:
return fmt.Errorf("unsupported types for '%s': input type %d, output type %d", name, typ.Input, typ.Output)
}
}
return nil
}
// Execute executes a PQL query.
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute")
@ -228,7 +190,6 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
} else if err := validateQueryContext(ctx); err != nil {
return resp, err
}
resp.Results = results
// Fill column attributes if requested.
@ -369,7 +330,6 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr
// handlePreCalls traverses the call tree looking for calls that need
// precomputed values. Right now, that's just Distinct.
func (e *executor) handlePreCalls(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
if c.Name == "Precomputed" {
idx := c.Args["valueidx"].(int64)
if idx >= 0 && idx < int64(len(opt.EmbeddedData)) {
@ -404,18 +364,17 @@ func (e *executor) handlePreCalls(ctx context.Context, tx Tx, index string, c *p
// like Distinct, where you can't predict output shard for a result
// from the shard being queried.
if newIndex != "" && newIndex != index {
if err := e.handlePreCallChildren(ctx, tx, index, c, shards, opt); err != nil {
return err
}
c.Type = pql.PrecallGlobal
index = newIndex
// we need to recompute shards, then
shards = nil
}
if err := e.handlePreCallChildren(ctx, tx, index, c, shards, opt); err != nil {
return err
}
// child calls already handled, no precall for this, so we're done
if c.Type == pql.PrecallNone {
// otherwise, handle the children
return e.handlePreCallChildren(ctx, tx, index, c, shards, opt)
return nil
}
// We don't try to handle sub-calls from here. I'm not 100%
// sure that's right, but I think the fact that they're happening
@ -548,6 +507,7 @@ func (e *executor) execute(ctx context.Context, tx Tx, index string, q *pql.Quer
if err != nil {
return nil, err
}
results = append(results, v)
// Some Calls can have significant data associated with them
// that gets generated during processing, such as Precomputed
@ -708,15 +668,6 @@ func (e *executor) executeCall(ctx context.Context, tx Tx, index string, c *pql.
return nil, err
}
// Special handling for mutation and top-n calls.
if op, ok := e.additionalCountOps[c.Name]; ok {
statFn()
return e.executeGenericCount(ctx, tx, index, c, op, shards, opt)
}
if op, ok := e.additionalFieldOps[c.Name]; ok {
statFn()
return e.executeGenericField(ctx, tx, index, c, op, shards, opt)
}
switch c.Name {
case "Sum":
statFn()
@ -739,6 +690,9 @@ func (e *executor) executeCall(ctx context.Context, tx Tx, index string, c *pql.
case "ClearRow":
statFn()
return e.executeClearRow(ctx, tx, index, c, shards, opt)
case "Distinct":
statFn()
return e.executeDistinct(ctx, tx, index, c, shards, opt)
case "Store":
statFn()
return e.executeSetRow(ctx, tx, index, c, shards, opt)
@ -1151,11 +1105,9 @@ func (e *executor) executeSum(ctx context.Context, tx Tx, index string, c *pql.C
return other, nil
}
// executeGenericField executes a generic call on a field. Note that in this
// implementation, the operation is always a BSI op.
func (e *executor) executeGenericField(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shards []uint64, opt *execOptions) (SignedRow, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericField")
span.LogKV("name", c.Name)
// executeDistinct executes a Distinct call on a field.
func (e *executor) executeDistinct(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (SignedRow, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinct")
defer span.Finish()
field := c.Args["field"]
@ -1165,7 +1117,7 @@ func (e *executor) executeGenericField(ctx context.Context, tx Tx, index string,
// Execute calls in bulk on each remote node and merge.
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeGenericFieldShard(ctx, tx, index, c, op, shard)
return e.executeDistinctShard(ctx, tx, index, c, shard)
}
// Merge returned results at coordinating node.
@ -1440,13 +1392,6 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index stri
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard")
defer span.Finish()
if _, ok := e.additionalCountOps[c.Name]; ok {
return nil, fmt.Errorf("count op %s used as bitmap call", c.Name)
}
if op, ok := e.additionalBitmapOps[c.Name]; ok {
return e.executeGenericBitmapShard(ctx, tx, index, c, op, shard)
}
switch c.Name {
case "Row", "Range":
return e.executeRowShard(ctx, tx, index, c, shard)
@ -1464,6 +1409,8 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index stri
return e.executeShiftShard(ctx, tx, index, c, shard)
case "All": // Allow a shard computation to use All() (note, limit/offset not applied)
return e.executeAllCallShard(ctx, tx, index, c, shard)
case "Distinct":
return nil, errors.New("Distinct shouldn't be hit as a bitmap call")
case "Precomputed":
return e.executePrecomputedCallShard(ctx, tx, index, c, shard)
default:
@ -1471,11 +1418,10 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index stri
}
}
// executeGenericFieldShard executes a generic/extension command on a
// single shard. Note that in this implementation, the op is always
// a BSI op.
func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shard uint64) (SignedRow, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericShard")
// executeDistinctShard executes a Distinct call on a single shard, yielding
// a SignedRow of the values found.
func (e *executor) executeDistinctShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (result SignedRow, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinctShard")
defer span.Finish()
var filter *Row
@ -1483,7 +1429,7 @@ func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index st
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard)
if err != nil {
return SignedRow{}, errors.Wrap(err, "executing bitmap call")
return result, errors.Wrap(err, "executing bitmap call")
}
filter = row
if filter != nil && len(filter.segments) > 0 {
@ -1497,29 +1443,115 @@ func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index st
field := e.Holder.Field(index, fieldName)
if field == nil {
return SignedRow{}, nil
return result, nil
}
bsig := field.bsiGroup(fieldName)
if bsig == nil {
return SignedRow{}, nil
return result, nil
}
view := viewBSIGroupPrefix + fieldName
depth := uint64(bsig.BitDepth)
offset := bsig.Base
existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*0, ShardWidth*1)
if err != nil {
return result, err
}
if filter != nil {
existsBitmap = existsBitmap.Intersect(filterBitmap)
}
if !existsBitmap.Any() {
return result, nil
}
fragment := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
if fragment == nil {
return SignedRow{}, nil
signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*1, ShardWidth*2)
if err != nil {
return result, nil
}
var out ext.SignedBitmap
if filterBitmap != nil {
out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{WrapBitmap(filterBitmap)}, c.Args)
} else {
out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{}, c.Args)
dataBitmaps := make([]*roaring.Bitmap, depth)
for i := uint64(0); i < depth; i++ {
dataBitmaps[i], err = tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*(i+2), ShardWidth*(i+3))
if err != nil {
return result, err
}
}
// we need spaces for sign bit, existence/filter bit, and data
// row bits, which we'll be grabbing 65k bits at a time
stashWords := make([]uint64, 1024*(depth+2))
bitStashes := make([][]uint64, depth)
for i := uint64(0); i < depth; i++ {
start := i * 1024
last := start + 1024
bitStashes[i] = stashWords[start:last]
i++
}
stashOffset := depth * 1024
existStash := stashWords[stashOffset : stashOffset+1024]
signStash := stashWords[stashOffset+1024 : stashOffset+2048]
dataBits := make([][]uint64, depth)
posValues := make([]uint64, 0, 64)
negValues := make([]uint64, 0, 64)
posBitmap := roaring.NewFileBitmap()
negBitmap := roaring.NewFileBitmap()
existIterator, _ := existsBitmap.Containers.Iterator(0)
for existIterator.Next() {
key, value := existIterator.Value()
if value.N() == 0 {
continue
}
exists := value.AsBitmap(existStash)
sign := signBitmap.Containers.Get(key).AsBitmap(signStash)
for i := uint64(0); i < depth; i++ {
dataBits[i] = dataBitmaps[i].Containers.Get(key).AsBitmap(bitStashes[i])
}
for idx, word := range exists {
// mask holds a mask we can test the other words against.
mask := uint64(1)
for word != 0 {
shift := uint(bits.TrailingZeros64(word))
// we shift one *more* than that, to move the
// actual one bit off.
word >>= shift + 1
mask <<= shift
value := int64(0)
for b := uint64(0); b < depth; b++ {
if dataBits[b][idx]&mask != 0 {
value += (1 << b)
}
}
if sign[idx]&mask != 0 {
value *= -1
}
value += int64(offset)
if value < 0 {
negValues = append(negValues, uint64(-value))
} else {
posValues = append(posValues, uint64(value))
}
// and now we processed that bit, so we move the mask over one.
mask <<= 1
}
if len(negValues) > 0 {
_, _ = negBitmap.AddN(negValues...)
negValues = negValues[:0]
}
if len(posValues) > 0 {
_, _ = posBitmap.AddN(posValues...)
posValues = posValues[:0]
}
}
}
return SignedRow{
Neg: NewRowFromBitmap(UnwrapBitmap(out.Neg)),
Pos: NewRowFromBitmap(UnwrapBitmap(out.Pos)),
Neg: NewRowFromBitmap(negBitmap),
Pos: NewRowFromBitmap(posBitmap),
}, nil
}
@ -2681,11 +2713,13 @@ func (e *executor) executeRowsShard(ctx context.Context, tx Tx, index string, fi
}
func (e *executor) executeRowShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowShard")
defer span.Finish()
// Handle bsiGroup ranges differently.
if c.HasConditionArg() {
// looks the same on badger/roaring. we think.
return e.executeRowBSIGroupShard(ctx, tx, index, c, shard)
}
@ -2795,6 +2829,7 @@ func (e *executor) executeRowShard(ctx context.Context, tx Tx, index string, c *
// executeRowBSIGroupShard executes a range(bsiGroup) call for a local shard.
func (e *executor) executeRowBSIGroupShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (_ *Row, err error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowBSIGroupShard")
defer span.Finish()
@ -2975,44 +3010,6 @@ func (e *executor) executeIntersectShard(ctx context.Context, tx Tx, index strin
return other, nil
}
// executeGenericBitmapShard executes a generic bitmap call for a local shard.
func (e *executor) executeGenericBitmapShard(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBitmap, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericBitmapShard")
defer span.Finish()
if op.BitmapOpArity() == ext.OpArityUnary {
if len(c.Children) != 1 {
return nil, fmt.Errorf("%s needs exactly one row parameter", c.Name)
}
row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard)
if err != nil {
return nil, err
}
return row.GenericUnaryOp(op.BitmapOpFunc(), c.Args), nil
}
var err error
rows := make([]*Row, len(c.Children))
for i, input := range c.Children {
rows[i], err = e.executeBitmapCallShard(ctx, tx, index, input, shard)
if err != nil {
return nil, err
}
}
var other *Row
switch op.BitmapOpArity() {
case ext.OpArityBinary:
other = rows[0]
for _, row := range rows[1:] {
other = other.GenericBinaryOp(op.BitmapOpFunc(), row, c.Args)
}
case ext.OpArityNary:
other = rows[0].GenericNaryOp(op.BitmapOpFunc(), rows[1:], c.Args)
}
other.invalidateCount()
return other, nil
}
// executeUnionShard executes a union() call for a local shard.
func (e *executor) executeUnionShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard")
@ -3164,41 +3161,6 @@ func (e *executor) executeShiftShard(ctx context.Context, tx Tx, index string, c
return row.Shift(n)
}
// executeGeneric executes a provided count-like call.
func (e *executor) executeGenericCount(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpUnaryCount, shards []uint64, opt *execOptions) (uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericCount")
defer span.Finish()
if len(c.Children) == 0 {
return 0, fmt.Errorf("%s() requires an input bitmap", c.Name)
} else if len(c.Children) > 1 {
return 0, fmt.Errorf("%s() only accepts a single bitmap input", c.Name)
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard)
if err != nil {
return 0, err
}
return row.GenericCount(op, c.Args), nil
}
// Merge returned results at coordinating node.
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(uint64)
return other + v.(uint64)
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return 0, err
}
n, _ := result.(uint64)
return n, nil
}
// executeCount executes a count() call.
func (e *executor) executeCount(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount")

34
executor_test.go
View file

@ -3413,6 +3413,7 @@ func TestExecutor_Execute_Existence(t *testing.T) {
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
@ -3438,6 +3439,7 @@ func TestExecutor_Execute_Existence(t *testing.T) {
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) {
t.Fatalf("unexpected columns after Not: %+v", bits)
}
// Reopen cluster to ensure existence field is reloaded.
if err := c[0].Reopen(); err != nil {
t.Fatal(err)
@ -4174,25 +4176,6 @@ func TestExecutor_Execute_SetRow(t *testing.T) {
t.Fatalf("unexpected columns: %+v", bits)
}
})
t.Run("Err_Store(Distinct)", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
f1, err := index.CreateField("f1", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
f2, err := index.CreateField("f2", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
q := fmt.Sprintf(`Store(Distinct(field=%s), %s=2)`, f1.Name(), f2.Name())
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: index.Name(), Query: q}); err == nil {
t.Fatalf("expected 'unsupported result type' error, got: %+v", res)
}
})
}
func benchmarkExistence(nn bool, b *testing.B) {
@ -5312,6 +5295,7 @@ func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOpt
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", *indexOptions)
defer index.Close()
_, err := index.CreateField("f", fieldOption...)
if err != nil {
t.Fatal(err)
@ -5723,12 +5707,15 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) {
if err := json.NewDecoder(bytes.NewReader(data)).Decode(schema); err != nil {
t.Fatal(err)
}
if err := api.ApplySchema(context.TODO(), schema, false); err != nil {
t.Fatal(err)
}
// AntitodePoint == row 1 b/c keys field.
writeQuery := `Set(100, type=AntidotePoint)Set(100, equip_id=100)Set(100, site_id=100)Set(100, id=100)`
for _, i := range schema.Indexes {
for k, i := range schema.Indexes {
_ = k
if _, err := api.Query(context.TODO(), &pilosa.QueryRequest{Index: i.Name, Query: writeQuery}); err != nil {
t.Fatal(err)
}
@ -5739,11 +5726,11 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) {
Intersect(
Distinct(
Intersect(Row(type=AntidotePoint)),
index=power_ts, field=equip_id),
index=equipment, field=equip_id),
Distinct(
Intersect(Row(type=AntidotePoint)),
index=power_ts, field=equip_id)
), index=equipment, field=site_id)`
index=sites, field=equip_id)
), index=power_ts, field=site_id)`
// Check if test query gives correct results (one column 100)
t.Run("Distinct", func(t *testing.T) {
@ -5760,6 +5747,7 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) {
}
if r.Pos.Count() != 1 {
t.Fatalf("invalid pilosa.SignedRow.Pos.Count, expected: 1, got: %v", r.Pos.Count())
}
if r.Pos.Columns()[0] != 100 {
t.Fatalf("invalid pilosa.SignedRow.Pos.Columns, expected: [100], got: %v", r.Pos.Columns())

96
extension.go
View file

@ -1,96 +0,0 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"github.com/molecula/ext"
"github.com/pilosa/pilosa/v2/roaring"
)
// WrapBitmap yields an extension-Bitmap from a roaring Bitmap.
func WrapBitmap(bm *roaring.Bitmap) ext.Bitmap {
return wrappedBitmap{bm}
}
// wrappedBitmap is a very shallow glue shim to convert a roaring Bitmap to
// an extension Bitmap.
type wrappedBitmap struct{ *roaring.Bitmap }
// UnwrapBitmap converts an extension-bitmap to its underlying roaring Bitmap.
func UnwrapBitmap(bm ext.Bitmap) *roaring.Bitmap {
if inner, ok := bm.(wrappedBitmap); ok {
if inner.Bitmap != nil {
return inner.Bitmap
}
return roaring.NewFileBitmap()
}
return roaring.NewFileBitmap()
}
func (b wrappedBitmap) Intersect(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Intersect(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) Union(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Union(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) IntersectionCount(other ext.Bitmap) uint64 {
return b.Bitmap.IntersectionCount(other.(wrappedBitmap).Bitmap)
}
func (b wrappedBitmap) Difference(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Difference(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) Xor(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Xor(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) Shift(n int) (ext.Bitmap, error) {
shifted, err := b.Bitmap.Shift(n)
return wrappedBitmap{shifted}, err
}
func (b wrappedBitmap) Flip(start, last uint64) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Flip(start, last)}
}
func (b wrappedBitmap) New() ext.Bitmap {
return WrapBitmap(roaring.NewFileBitmap())
}
// ContainerBits tries to get one container's worth of bits.
func (b wrappedBitmap) ContainerBits(offset uint64, target []uint64) (out []uint64) {
// it's an error to call this with a non-container-aligned offset
if offset&0xFFFF != 0 {
return nil
}
if b.Bitmap == nil {
fmt.Printf("ContainerBits on bitmap with no contents\n")
return nil
}
if b.Bitmap.Containers == nil {
fmt.Printf("ContainerBits on bitmap with nil Containers\n")
return nil
}
c := b.Bitmap.Containers.Get(offset >> 16)
if c == nil {
return nil
}
return c.AsBitmap(target)
}

20
field.go
View file

@ -92,6 +92,8 @@ type Field struct {
name string
qualifiedName string
idx *Index
viewMap map[string]*view
// Row attribute storage and cache
@ -1181,6 +1183,7 @@ func (f *Field) createViewIfNotExistsBase(name string) (*view, bool, error) {
func (f *Field) newView(path, name string) *view {
view := newView(f.holder, path, f.index, f.name, name, f.options)
view.idx = f.idx
view.rowAttrStore = f.rowAttrStore
view.stats = f.Stats
view.broadcaster = f.broadcaster
@ -1432,6 +1435,14 @@ func (f *Field) SetValue(tx Tx, columnID uint64, value int64) (changed bool, err
if err != nil {
return false, errors.Wrap(err, "creating view")
}
if view.holder == nil {
panic("view.holder should not be nil")
}
if view.idx == nil {
panic("view.idx should not be nil")
}
view.holder.addIndexFromField(view.idx)
return view.setValue(tx, columnID, bsig.BitDepth, baseValue)
}
@ -1761,6 +1772,10 @@ func (f *Field) importRoaring(ctx context.Context, tx Tx, data []byte, shard uin
return nil
}
func (f *Field) GetIndex() *Index {
return f.idx
}
func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte, shard uint64, viewName string, block int) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Field.importRoaringOverwrite")
defer span.Finish()
@ -1787,7 +1802,7 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte,
switch f.Options().Type {
case FieldTypeInt, FieldTypeDecimal:
frag.mu.Lock()
if err := frag.calculateMaxRowID(); err != nil {
if err := frag.calculateMaxRowID(tx); err != nil {
return err
}
maxRowID, _, err := frag.maxRow(tx, nil)
@ -2132,6 +2147,9 @@ func isValidCacheType(v string) bool {
}
}
// TODO(jea): why isn't this bits.Len64(x) using import "math/bits"
// That would be much (80x or more) faster and correct if the high bit is set.
//
// bitDepth returns the number of bits required to store a value.
func bitDepth(v uint64) uint {
for i := uint(0); i < 63; i++ {

58
field_internal_test.go
View file

@ -205,10 +205,17 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField {
if err != nil {
t.Fatal(err)
}
field, err := NewField(NewHolder(DefaultPartitionN), path, "i", "f", opts)
h := NewHolder(DefaultPartitionN)
h.Path = path
idx, err := h.CreateIndex("i", IndexOptions{})
if err != nil {
panic(err)
}
field, err := NewField(h, path, "i", "f", opts)
if err != nil {
t.Fatal(err)
}
field.idx = idx
return &TestField{Field: field}
}
@ -223,6 +230,9 @@ func OpenField(t *testing.T, opts FieldOption) *TestField {
// Close closes the field and removes the underlying data.
func (f *TestField) Close() error {
if f.idx != nil {
panicOn(f.idx.Txf.CloseIndex(f.idx))
}
defer os.RemoveAll(f.Path())
return f.Field.Close()
}
@ -235,10 +245,17 @@ func (f *TestField) Reopen() error {
}
path, index, name := f.Path(), f.Index(), f.Name()
f.Field, err = NewField(NewHolder(DefaultPartitionN), path, index, name, OptFieldTypeDefault())
h := NewHolder(DefaultPartitionN)
h.Path = path
idx, err := h.CreateIndex(index, IndexOptions{})
if err != nil {
return err
}
f.Field, err = NewField(h, path, index, name, OptFieldTypeDefault())
if err != nil {
return err
}
f.Field.idx = idx
if err := f.Open(); err != nil {
return err
@ -311,7 +328,8 @@ func TestField_RowTime(t *testing.T) {
defer f.Close()
// Obtain transaction.
tx := &RoaringTx{Field: f.Field}
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
@ -323,6 +341,12 @@ func TestField_RowTime(t *testing.T) {
f.MustSetBit(tx, 1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC))
f.MustSetBit(tx, 1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC))
panicOn(tx.Commit())
// obtain 2nd transaction to read it back.
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
if r, err := f.RowTime(tx, 1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(r.Columns(), []uint64{1, 3, 4, 5}) {
@ -358,6 +382,7 @@ func TestField_RowTime(t *testing.T) {
func TestField_PersistAvailableShards(t *testing.T) {
availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
f := OpenField(t, OptFieldTypeDefault())
defer f.Close()
// bm represents remote available shards.
bm := roaring.NewBitmap(1, 2, 3)
@ -379,6 +404,7 @@ func TestField_PersistAvailableShards(t *testing.T) {
func TestField_CorruptAvailableShards(t *testing.T) {
availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
f := OpenField(t, OptFieldTypeDefault())
defer f.Close()
// bm represents remote available shards.
bm := roaring.NewBitmap(1, 2, 3)
@ -411,6 +437,7 @@ func TestField_CorruptAvailableShards(t *testing.T) {
func TestField_TruncatedAvailableShards(t *testing.T) {
availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
f := OpenField(t, OptFieldTypeDefault())
defer f.Close()
// bm represents remote available shards.
bm := roaring.NewBitmap(1, 2, 3)
@ -441,6 +468,7 @@ func TestField_TruncatedAvailableShards(t *testing.T) {
func TestField_PersistAvailableShardsFootprint(t *testing.T) {
availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
f := OpenField(t, OptFieldTypeDefault())
defer f.Close()
// bm represents remote available shards.
bm := roaring.NewBitmap()
@ -554,6 +582,7 @@ func TestField_ApplyOptions(t *testing.T) {
// to result in a value of 9 instead of 1.
func TestBSIGroup_importValue(t *testing.T) {
f := OpenField(t, OptFieldTypeInt(-100, 200))
defer f.Close()
options := &ImportOptions{}
for i, tt := range []struct {
@ -581,12 +610,17 @@ func TestBSIGroup_importValue(t *testing.T) {
[]uint64{100},
},
} {
tx := &RoaringTx{Field: f.Field}
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
if err := f.importValue(tx, tt.columnIDs, tt.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(tx.Commit())
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
if row, err := f.Range(tx, f.name, pql.EQ, tt.checkVal); err != nil {
t.Fatalf("test %d, getting range: %s", i, err.Error())
} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
@ -597,6 +631,7 @@ func TestBSIGroup_importValue(t *testing.T) {
func TestIntField_MinMaxForShard(t *testing.T) {
f := OpenField(t, OptFieldTypeInt(-100, 200))
defer f.Close()
options := &ImportOptions{}
for i, test := range []struct {
@ -648,12 +683,17 @@ func TestIntField_MinMaxForShard(t *testing.T) {
},
} {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
tx := &RoaringTx{Field: f.Field}
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
if err := f.importValue(tx, test.columnIDs, test.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(tx.Commit())
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
maxvc, err := f.MaxForShard(tx, 0, nil)
if err != nil {
t.Fatalf("getting max for shard: %v", err)
@ -753,6 +793,7 @@ func TestDecimalField_MinMaxBoundaries(t *testing.T) {
func TestDecimalField_MinMaxForShard(t *testing.T) {
f := OpenField(t, OptFieldTypeDecimal(3))
defer f.Close()
options := &ImportOptions{}
for i, test := range []struct {
@ -804,12 +845,17 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
},
} {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
tx := &RoaringTx{Field: f.Field}
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
if err := f.importFloatValue(tx, test.columnIDs, test.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(tx.Commit())
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
maxvc, err := f.MaxForShard(tx, 0, nil)
if err != nil {
t.Fatalf("getting max for shard: %v", err)

40
field_test.go
View file

@ -25,6 +25,8 @@ import (
"github.com/pilosa/pilosa/v2/test"
)
var panicOn = pilosa.PanicOn
// Ensure a field can set & read a bsiGroup value.
func TestField_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
@ -35,7 +37,10 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxPilosa := f.Field.GetIndex()
tx := idxPilosa.NewTx(pilosa.Txo{Write: writable, Index: idxPilosa, Field: f.Field})
defer tx.Rollback()
// Set value on field.
if changed, err := f.SetValue(tx, 100, 21); err != nil {
@ -69,7 +74,9 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Set value.
if changed, err := f.SetValue(tx, 100, 21); err != nil {
@ -103,7 +110,9 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Set value.
if _, err := f.SetValue(tx, 100, 21); err != pilosa.ErrBSIGroupNotFound {
@ -119,7 +128,9 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Set value.
if _, err := f.SetValue(tx, 100, 15); err != pilosa.ErrBSIGroupValueTooLow {
@ -135,7 +146,9 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Set value.
if _, err := f.SetValue(tx, 100, 31); err != pilosa.ErrBSIGroupValueTooHigh {
@ -204,7 +217,9 @@ func TestField_AvailableShards(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Set values on shards 0 & 2, and verify.
if _, err := f.SetBit(tx, 0, 100, nil); err != nil {
@ -214,6 +229,7 @@ func TestField_AvailableShards(t *testing.T) {
} else if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" {
t.Fatal(diff)
}
panicOn(tx.Commit())
// Set remote shards and verify.
if err := f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)); err != nil {
@ -244,7 +260,9 @@ func TestField_ClearValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Set value on field.
if changed, err := f.SetValue(tx, 100, 21); err != nil {
@ -252,6 +270,9 @@ func TestField_ClearValue(t *testing.T) {
} else if !changed {
t.Fatal("expected change")
}
panicOn(tx.Commit())
tx = idxP.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idxP, Field: f.Field})
// Read value.
if value, exists, err := f.Value(tx, 100); err != nil {
@ -261,12 +282,17 @@ func TestField_ClearValue(t *testing.T) {
} else if !exists {
t.Fatal("expected value to exist")
}
tx.Rollback()
tx = idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
if changed, err := f.ClearValue(tx, 100); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal(err)
}
panicOn(tx.Commit())
tx = idxP.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Read value.
if _, exists, err := f.Value(tx, 100); err != nil {

169
fragment.go
View file

@ -108,6 +108,9 @@ type fragment struct {
view string
shard uint64
// idx cached to avoid repeatedly looking it up everywhere.
idx *Index
// parent holder, used to find snapshot queue, etc.
holder *Holder
@ -180,6 +183,10 @@ type fragment struct {
// newFragment returns a new instance of Fragment.
func newFragment(holder *Holder, path, index, field, view string, shard uint64, flags byte) *fragment {
idx := holder.Index(index)
if idx == nil {
panic(fmt.Sprintf("holder=%#v but got nil idx back from holder!", holder))
}
f := &fragment{
path: path,
index: index,
@ -187,6 +194,7 @@ func newFragment(holder *Holder, path, index, field, view string, shard uint64,
view: view,
shard: shard,
flags: flags,
idx: idx,
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
@ -250,7 +258,9 @@ func (f *fragment) Open() error {
f.checksums = make(map[int][]byte)
// Read last bit to determine max row.
return f.calculateMaxRowID()
tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f})
defer tx.Rollback()
return f.calculateMaxRowID(tx)
}(); err != nil {
f.close()
return err
@ -398,6 +408,15 @@ func (f *fragment) inspectStorage(data []byte, file *os.File, newGen generation,
// logic is now mostly in importStorage (reading in a bitmap) and applyStorage
// (remapping an existing bitmap to match a new backing store).
func (f *fragment) openStorage(unmarshalData bool) error {
if !f.idx.NeedsSnapshot() {
f.gen = &NopGeneration{}
f.rowCache = &simpleCache{make(map[uint64]*Row)}
f.currdata = struct{ from, to uintptr }{}
f.prevdata = f.currdata
return nil // openStorage becomes a noop under RBF, Badger, etc.
}
// Create a roaring bitmap to serve as storage for the shard.
if f.storage == nil {
f.storage = roaring.NewFileBitmap()
@ -473,10 +492,16 @@ func (f *fragment) openCache() error {
return nil
}
tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f})
defer tx.Rollback()
// Read in all rows by ID.
// This will cause them to be added to the cache.
for _, id := range pb.IDs {
n := f.storage.CountRange(id*ShardWidth, (id+1)*ShardWidth)
n, err := tx.CountRange(f.index, f.field, f.view, f.shard, id*ShardWidth, (id+1)*ShardWidth)
if err != nil {
return errors.Wrap(err, "CountRange")
}
f.cache.BulkAdd(id, n)
}
f.cache.Invalidate()
@ -583,7 +608,7 @@ func (f *fragment) rowFromStorage(tx Tx, rowID uint64) (*Row, error) {
row := &Row{
segments: []rowSegment{{
data: data, // this data contains BadgerTx data, which should not survive Txn commit.
data: data,
shard: f.shard,
writable: true,
}},
@ -598,7 +623,11 @@ func (f *fragment) rowFromStorage(tx Tx, rowID uint64) (*Row, error) {
func (f *fragment) setBit(tx Tx, rowID, columnID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
// handle mutux field type
if f.mutexVector != nil {
if err := f.handleMutex(tx, rowID, columnID); err != nil {
@ -680,7 +709,11 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo
func (f *fragment) clearBit(tx Tx, rowID, columnID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
changed, err = f.unprotectedClearBit(tx, rowID, columnID)
return err
})
@ -739,7 +772,11 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b
func (f *fragment) setRow(tx Tx, row *Row, rowID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
changed, err = f.unprotectedSetRow(tx, row, rowID)
return err
})
@ -802,7 +839,11 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo
func (f *fragment) clearRow(tx Tx, rowID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
changed, err = f.unprotectedClearRow(tx, rowID)
return err
})
@ -845,7 +886,11 @@ func (f *fragment) unprotectedClearRow(tx Tx, rowID uint64) (changed bool, err e
// This updates both the on-disk storage and the in-cache bitmap.
func (f *fragment) unprotectedClearBlock(tx Tx, block int) (changed bool, err error) {
firstRow := uint64(block * HashBlockSize)
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
var rowChanged bool
for rowID := uint64(firstRow); rowID < firstRow+HashBlockSize; rowID++ {
if changed, err := f.unprotectedClearRow(tx, rowID); err != nil {
@ -953,8 +998,11 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64
func (f *fragment) setValueBase(tx Tx, columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
// Convert value to an unsigned representation.
uvalue := uint64(value)
if value < 0 {
@ -1291,8 +1339,12 @@ func (f *fragment) maxRow(tx Tx, filter *Row) (uint64, uint64, error) {
// calculateMaxRowID determines the field's maxRowID value based
// on the contents of its storage, and sets the struct argument.
func (f *fragment) calculateMaxRowID() (err error) {
f.maxRowID = f.storage.Max() / ShardWidth
func (f *fragment) calculateMaxRowID(tx Tx) (err error) {
max, err := tx.Max(f.index, f.field, f.view, f.shard)
if err != nil {
return err
}
f.maxRowID = max / ShardWidth
return nil
}
@ -1484,7 +1536,6 @@ func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality
if err != nil {
return nil, err
}
// Create predicate without sign bit.
upredicate := absInt64(predicate)
@ -1492,7 +1543,6 @@ func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality
if err != nil {
return nil, err
}
switch {
case predicate == 0 && !allowEquality:
// Match all positive numbers except zero.
@ -2217,7 +2267,11 @@ func (f *fragment) bulkImportStandard(tx Tx, rowIDs, columnIDs []uint64, options
// operations to the op log.
func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64]struct{}) error {
//tx.AddN()
err := f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err := f.gen.Transaction(wp, func() error { // segfault
if len(set) > 0 {
f.stats.Count(MetricImportingN, int64(len(set)), 1)
@ -2453,14 +2507,18 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b
span, ctx := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits")
var changed int
var rowSet map[uint64]int
err := f.gen.Transaction(&f.storage.OpWriter, func() (err error) {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err := f.gen.Transaction(wp, func() (err error) {
var rit roaring.RoaringIterator
rit, err = roaring.NewRoaringIterator(data)
if err != nil {
return err
}
changed, rowSet, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, rit, clear, true, rowSize)
changed, rowSet, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, rit, clear, true, rowSize, nil)
return err
})
@ -2550,6 +2608,9 @@ func track(start time.Time, message string, stats stats.StatsClient, logger logg
// snapshot does the actual snapshot operation. it does not check or care
// about f.snapshotPending.
func (f *fragment) snapshot() (err error) {
if !f.idx.NeedsSnapshot() {
return nil
}
if !f.open {
return errors.New("snapshot request on closed fragment")
}
@ -2688,31 +2749,16 @@ func (f *fragment) WriteTo(w io.Writer) (n int64, err error) {
return 0, nil
}
// used in shipping the slices across the network for a resize.
func (f *fragment) writeStorageToArchive(tw *tar.Writer) error {
// Open separate file descriptor to read from.
file, err := os.Open(f.path)
tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx})
defer tx.Rollback()
file, sz, err := tx.RoaringBitmapReader(f.index, f.field, f.view, f.shard, f.path)
if err != nil {
return errors.Wrap(err, "opening file")
}
defer file.Close()
// Retrieve the current file size under lock so we don't read
// while an operation is appending to the end.
var sz int64
if err := func() error {
f.mu.Lock()
defer f.mu.Unlock()
fi, err := file.Stat()
if err != nil {
return errors.Wrap(err, "statting")
}
sz = fi.Size()
return nil
}(); err != nil {
return err
}
defer file.Close()
// Write archive header.
if err := tw.WriteHeader(&tar.Header{
@ -2779,9 +2825,15 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) {
// Process file based on file name.
switch hdr.Name {
case "data":
if err := f.readStorageFromArchive(tr); err != nil {
idx := f.holder.Index(f.index)
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
defer tx.Rollback()
if err := f.fillFragmentFromArchive(tx, tr); err != nil {
return 0, errors.Wrap(err, "reading storage")
}
if err := tx.Commit(); err != nil {
return 0, errors.Wrap(err, "Commit after tx.ReadFragmentFromArchive")
}
case "cache":
if err := f.readCacheFromArchive(tr); err != nil {
return 0, errors.Wrap(err, "reading cache")
@ -2794,7 +2846,40 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) {
return 0, nil
}
// should be morally equivalent to fragment.readStorageFromArchive()
// below for RoaringTx, but also work on any Tx because it uses
// tx.ImportRoaringBits().
func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error {
// this is reading from inside a tarball, so definitely no need
// to close it here.
data, err := ioutil.ReadAll(r)
if err != nil {
return errors.Wrap(err, "fillFragmentFromArchive ioutil.ReadAll(r)")
}
if len(data) == 0 {
return nil
}
// For reference, compare to what fragment.go:313 fragment.importStorage() does.
clear := false
log := false
rowSize := uint64(0)
itr, err := roaring.NewRoaringIterator(data)
if err != nil {
return errors.Wrap(err, "fillFragmentFromArchive NewRoaringIterator")
}
changed, rowSet, err := tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, itr, clear, log, rowSize, data)
_, _ = changed, rowSet
if err != nil {
return errors.Wrap(err, "fillFragmentFromArchive ImportRoaringBits")
}
return nil
}
func (f *fragment) readStorageFromArchive(r io.Reader) error {
// Create a temporary file to copy into.
path := f.path + copyExt
file, err := os.Create(path)
@ -2808,6 +2893,12 @@ func (f *fragment) readStorageFromArchive(r io.Reader) error {
return errors.Wrap(err, "copying")
}
// TODO(jea): isn't this next Rename a file handle leak?
// try closing first
if err := f.closeStorage(); err != nil {
return errors.Wrap(err, "closeStorage-prior-to-Rename-and-openStorage")
}
// Move snapshot to data file location.
if err := os.Rename(path, f.path); err != nil {
return errors.Wrap(err, "renaming")

1055
fragment_internal_test.go
View file

File diff suppressed because it is too large Load diff

23
generation.go
View file

@ -261,6 +261,7 @@ func (m *mmapGeneration) openFile() (shouldClose bool, err error) {
if err != nil {
return false, err
}
// do we actually want this in every openFile? I don't know.
if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
_ = syswrap.CloseFile(m.file)
@ -438,3 +439,25 @@ func newGeneration(existing generation, path string, readData bool, setup func([
// does get cleaned up.
return &m, nil
}
// NopGeneration is used in fragment.openStorage() to short-circuit
// generation stuff that only applies to RoaringTx; doesn't apply to RBFTx/BadgerTx/etc.
type NopGeneration struct {
}
func (g *NopGeneration) Transaction(w *io.Writer, f func() error) error {
return f()
}
func (g *NopGeneration) Done() {}
func (g *NopGeneration) Generation() int64 {
return 0
}
func (g *NopGeneration) ID() string {
return "NOP"
}
func (g *NopGeneration) Dead() bool {
return true
}
func (g *NopGeneration) Bytes() (ret []byte) {
return
}

7
generation_test.go
View file

@ -53,8 +53,11 @@ func TestGenerationPanic(t *testing.T) {
if unsafe.Pointer(&prevData[0]) == unsafe.Pointer(&newData[0]) {
t.Fatalf("test can't run usefully, didn't get new data pointer")
}
err := f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err := f.gen.Transaction(wp, func() error {
prevData[0] = 0x3c
return nil
})

48
holder.go
View file

@ -528,9 +528,9 @@ func (h *Holder) Open() error {
if h.isCoordinator() {
index.createdAt = timestamp()
err = index.OpenWithTimestamp()
err = index.OpenWithTimestamp(false)
} else {
err = index.Open()
err = index.Open(false)
}
if err != nil {
if err == ErrName {
@ -626,6 +626,17 @@ func (h *Holder) BeginTx(writable bool, index *Index) (Tx, error) {
return index.Txf.NewTx(Txo{Write: writable, Index: index}), nil
}
func (h *Holder) NeedsSnapshot() bool {
h.mu.RLock()
defer h.mu.RUnlock()
for _, idx := range h.indexes {
if idx.NeedsSnapshot() {
return true
}
}
return false
}
// HasData returns true if Holder contains at least one index.
// This is used to determine if the rebalancing of data is necessary
// when a node joins the cluster.
@ -802,7 +813,20 @@ func (h *Holder) applyCreatedAt(indexes []*IndexInfo) {
}
// IndexPath returns the path where a given index is stored.
func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
func (h *Holder) IndexPath(name string) string {
return filepath.Join(h.Path, name)
}
// HolderPathFromIndexPath is
// used by test/index.go:71 in test.Index.Reopen() to get the right
// path into a test Holder that doesn't know its own proper path.
// If the Holder changes index paths to being something other than
// holderPath + "/" + indexName, this will need adjusting too.
func (h *Holder) HolderPathFromIndexPath(indexPath, indexName string) string {
n := len(indexPath)
hpath2 := indexPath[:n-(len(indexName)+1)]
return hpath2
}
// Index returns the index by name.
func (h *Holder) Index(name string) *Index {
@ -811,7 +835,9 @@ func (h *Holder) Index(name string) *Index {
return h.index(name)
}
func (h *Holder) index(name string) *Index { return h.indexes[name] }
func (h *Holder) index(name string) *Index {
return h.indexes[name]
}
// Indexes returns a list of all indexes in the holder.
func (h *Holder) Indexes() []*Index {
@ -867,7 +893,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
index.keys = opt.Keys
index.trackExistence = opt.TrackExistence
if err = index.Open(); err != nil {
if err = index.Open(true); err != nil {
return nil, errors.Wrap(err, "opening")
}
if err = index.saveMeta(); err != nil {
@ -1799,3 +1825,15 @@ func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) {
}
return nil
}
// used by Index.openFields(), enabling Tx / Txf by telling
// the holder about its own indexes.
func (h *Holder) addIndexFromField(idx *Index) {
h.mu.Lock()
h.indexes[idx.Name()] = idx
h.mu.Unlock()
}
func (h *Holder) unprotectedAddIndexFromField(idx *Index) {
h.indexes[idx.Name()] = idx
}

22
holder_test.go
View file

@ -32,6 +32,8 @@ import (
)
func TestHolder_Open(t *testing.T) {
skipForBadger := os.Getenv("PILOSA_TXSRC") == "badger"
t.Run("ErrIndexName", func(t *testing.T) {
h := test.MustOpenHolder()
@ -165,6 +167,9 @@ func TestHolder_Open(t *testing.T) {
})
t.Run("ErrFragmentStoragePermission", func(t *testing.T) {
if skipForBadger {
t.Skip("skipping for badger")
}
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
@ -201,6 +206,10 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) {
if skipForBadger {
t.Skip("skipping for badger")
}
h := test.MustOpenHolder()
defer h.Close()
@ -233,6 +242,10 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
if skipForBadger {
t.Skip("skipping for badger")
}
h := test.MustOpenHolder()
defer h.Close()
@ -723,24 +736,17 @@ func TestHolderSyncer_IntField(t *testing.T) {
hldr0.SetValue("i", "f", 1, 1)
// in c0 expect the 1 bit
//idx0.Dump("in c0, before SyncData")
// Set data on node1. columnID=2, value=2
idx1 := hldr1.SetValue("i", "f", 2, 2)
_ = idx1
//idx1.Dump("in c1, before SyncData")
//vv("before c[0] SyncData")
err = c[0].Server.SyncData()
if err != nil {
t.Fatalf("syncing node 0: %v", err)
}
//vv("after c[0] SyncData")
// expect 3 rows, the 1 bit + 2 rows for the 2 value as BSI. But, we only see that c0 overwrote c1.
//idx0.Dump("in c0, after syncData")
//idx1.Dump("in c1, after syncData")
// Problem is: data at c1 was replaced by c0, instead of being merged with existing c1.
// Problem is: data at c0 did not receive and merge the c1 data.
@ -810,8 +816,6 @@ func TestHolderSyncer_IntField(t *testing.T) {
}
// dump the badger keys for both c0 and c1
//vv("in c0, allkeys = '%v'", idx0.StringifiedBadgerKeys(nil))
//vv("in c1, allkeys = '%v'", c[1].index.StringifiedBadgerKeys())
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {

63
index.go
View file

@ -99,16 +99,16 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
}
}
txf, err := newTxFactory(txsrc, path)
if err != nil {
return nil, errors.Wrap(err, "creating newTxFactory")
}
err = validateName(name)
err := validateName(name)
if err != nil {
return nil, errors.Wrap(err, "validating name")
}
txf, err := NewTxFactory(txsrc, holder.Path, name)
if err != nil {
return nil, errors.Wrap(err, "creating newTxFactory")
}
idx := &Index{
path: path,
name: name,
@ -134,6 +134,14 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
return idx, nil
}
func (i *Index) NewTx(txo Txo) Tx {
return i.Txf.NewTx(txo)
}
func (i *Index) NeedsSnapshot() bool {
return i.Txf.NeedsSnapshot()
}
// CreatedAt is an timestamp for a specific version of an index.
func (i *Index) CreatedAt() int64 {
i.mu.RLock()
@ -148,7 +156,9 @@ func (i *Index) Name() string { return i.name }
func (i *Index) QualifiedName() string { return i.qualifiedName }
// Path returns the path the index was initialized with.
func (i *Index) Path() string { return i.path }
func (i *Index) Path() string {
return i.path
}
// TranslateStorePath returns the translation database path for a partition.
func (i *Index) TranslateStorePath(partitionID int) string {
@ -181,12 +191,12 @@ func (i *Index) options() IndexOptions {
}
// Open opens and initializes the index.
func (i *Index) Open() error { return i.open(false) }
func (i *Index) Open(haveHolderLock bool) error { return i.open(false, haveHolderLock) }
// OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields.
func (i *Index) OpenWithTimestamp() error { return i.open(true) }
func (i *Index) OpenWithTimestamp(haveHolderLock bool) error { return i.open(true, haveHolderLock) }
func (i *Index) open(withTimestamp bool) (err error) {
func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) {
// Ensure the path exists.
i.holder.Logger.Debugf("ensure index path exists: %s", i.path)
if err := os.MkdirAll(i.path, 0777); err != nil {
@ -200,7 +210,7 @@ func (i *Index) open(withTimestamp bool) (err error) {
}
i.holder.Logger.Debugf("open fields for index: %s", i.name)
if err := i.openFields(withTimestamp); err != nil {
if err := i.openFields(withTimestamp, haveHolderLock); err != nil {
return errors.Wrap(err, "opening fields")
}
@ -243,7 +253,7 @@ func (i *Index) open(withTimestamp bool) (err error) {
var indexQueue = make(chan struct{}, 8)
// openFields opens and initializes the fields inside the index.
func (i *Index) openFields(withTimestamp bool) error {
func (i *Index) openFields(withTimestamp, haveHolderLock bool) error {
f, err := os.Open(i.path)
if err != nil {
return errors.Wrap(err, "opening directory")
@ -273,7 +283,31 @@ fileLoop:
<-indexQueue
}()
i.holder.Logger.Debugf("open field: %s", fi.Name())
mu.Lock()
// i.holder needs to know about its index i for the Txf to work.
//
// We face either a deadlock or a race here.
//
// We get a deadlock in TestIndex_CreateField/"BSIFields"/"OK"
// if we call addIndexFromField, because in that test
// we get here while already holding i.holder.mu.
//
// On the other had, we get races on other tests
// such as TestExecutor_Execute_Existence/Row
// if we call unprotectedAddIndexFromField which does
// not lock i.holder.mu.
//
// The resolution was to have the goroutines that are holding
// the lock already tell us. That is the haveHolderLock
// argument.
if haveHolderLock {
i.holder.unprotectedAddIndexFromField(i)
} else {
i.holder.addIndexFromField(i)
}
fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if withTimestamp {
fld.createdAt = timestamp()
@ -287,6 +321,7 @@ fileLoop:
// up a foreign index.
fld.holder = i.holder
// open all the views
if err := fld.Open(); err != nil {
return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
}
@ -546,6 +581,9 @@ func (i *Index) createField(name string, opt *FieldOptions) (*Field, error) {
// Add to index's field lookup.
i.fields[name] = f
// enable Txf to find the index in field_test.go TestField_SetValue
f.idx = i
// Kick off the field's translation sync process.
if err := i.translationSyncer.Reset(); err != nil {
return nil, errors.Wrap(err, "resetting translation syncer")
@ -559,6 +597,7 @@ func (i *Index) newField(path, name string) (*Field, error) {
if err != nil {
return nil, err
}
f.idx = i
f.Stats = i.Stats
f.broadcaster = i.broadcaster
f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data"))

9
index_internal_test.go
View file

@ -25,7 +25,10 @@ func mustOpenIndex(opt IndexOptions) *Index {
if err != nil {
panic(err)
}
index, err := NewIndex(NewHolder(1), path, "i")
h := NewHolder(1)
h.Path = path
index, err := h.CreateIndex("i", opt)
if err != nil {
panic(err)
}
@ -33,7 +36,7 @@ func mustOpenIndex(opt IndexOptions) *Index {
index.keys = opt.Keys
index.trackExistence = opt.TrackExistence
if err := index.Open(); err != nil {
if err := index.Open(false); err != nil {
panic(err)
}
return index
@ -44,7 +47,7 @@ func (i *Index) reopen() error {
if err := i.Close(); err != nil {
return err
}
if err := i.Open(); err != nil {
if err := i.Open(false); err != nil {
return err
}
return nil

4
mmap_test.go
View file

@ -32,11 +32,11 @@ type cv struct {
func forceSnapshotsCheckMapping(t *testing.T) {
depth := uint(6)
f, idx := mustOpenBSIFragment("i", "f", viewStandard, 0)
_ = idx
f.Logger = logger.NewLogfLogger(t)
defer f.Clean(t)
tx := &RoaringTx{fragment: f}
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
defer tx.Rollback()
for i := 0; i < f.MaxOpN; i++ {
_, _ = f.setBit(tx, 0, uint64(32*i))

34
pql/ast.go
View file

@ -22,8 +22,6 @@ import (
"strconv"
"strings"
"time"
"github.com/molecula/ext"
)
// Query represents a PQL query.
@ -345,7 +343,8 @@ var callInfoByFunc = map[string]callInfo{
"Row": {allowUnknown: true},
"Range": {allowUnknown: true},
"Distinct": {allowUnknown: true},
"Distinct": {allowUnknown: true, callType: PrecallGlobal},
"Condition": {allowUnknown: true},
// allow only "field=X" cases with string field names
"Max": allowField,
@ -462,30 +461,6 @@ var callInfoByFunc = map[string]callInfo{
},
}
// RegisterPluginFuncs adds arg validation for plugin funcs. Not very good
// arg validation.
func RegisterPluginFuncs(ops []ext.BitmapOp) {
for _, op := range ops {
// ignore overlap for now. This should change.
if _, ok := callInfoByFunc[op.Name]; ok {
continue
}
ci := callInfo{allowUnknown: true}
if len(op.Reserved) > 0 {
// mark these as valid/known reserved words
ci.prototypes = make(map[string]interface{})
for _, res := range op.Reserved {
ci.prototypes[res] = nil
}
}
t := op.Func.BitmapOpType()
if t.Precall == ext.OpPrecallGlobal {
ci.callType = PrecallGlobal
}
callInfoByFunc[op.Name] = ci
}
}
// CheckCallInfo tries to validate that arguments are correct and valid for the
// given call. It does not guarantee checking all possible errors; for instance,
// if an argument is a field name, CheckCallInfo can't validate that the field
@ -505,6 +480,11 @@ func (c *Call) CheckCallInfo() error {
if !ok && strings.HasPrefix(k, "_") {
return fmt.Errorf("'%s': unknown reserved arg '%s'", c.String(), k)
}
if call, ok := v.(*Call); ok {
if err := call.CheckCallInfo(); err != nil {
return err
}
}
if acceptable == nil {
continue
}

4
rbf/db.go
View file

@ -71,6 +71,10 @@ func NewDBWithShard(path string, shard int) *DB {
}
}
func (db *DB) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
panic("TODO: implement rbf.DB.DeleteFragment")
}
// DataPath returns the path to the data file for the DB.
func (db *DB) DataPath() string {
return filepath.Join(db.Path, "data")

6
roaring/container_stash.go
View file

@ -387,7 +387,7 @@ func (c *Container) bitmap() []uint64 {
// is provided. The target should be zeroed, or this becomes an implicit
// union.
func (c *Container) AsBitmap(target []uint64) (out []uint64) {
if c.typeID == containerBitmap {
if c != nil && c.typeID == containerBitmap {
return c.bitmap()
}
// Reminder: len(nil) == 0.
@ -399,6 +399,10 @@ func (c *Container) AsBitmap(target []uint64) (out []uint64) {
out[i] = 0
}
}
// A nil *Container is a valid empty container.
if c == nil {
return out
}
if c.typeID == containerArray {
a := c.array()
for _, v := range a {

118
row.go
View file

@ -18,7 +18,6 @@ import (
"encoding/json"
"sort"
"github.com/molecula/ext"
pb "github.com/pilosa/pilosa/v2/proto"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
@ -326,65 +325,6 @@ func (r *Row) Union(others ...*Row) *Row {
return &Row{segments: output}
}
// GenericBinaryOp returns the output of a generic op on r and other.
func (r *Row) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *Row, args map[string]interface{}) *Row {
var segments []rowSegment
itr := newMergeSegmentIterator(r.segments, other.segments)
for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
if s1 == nil {
segments = append(segments, *s0)
continue
} else if s0 == nil {
segments = append(segments, *s1)
continue
}
segments = append(segments, *s0.GenericBinaryOp(op, s1, args))
}
return &Row{segments: segments}
}
// GenericNaryOp returns the output of an nary op on r and others.
func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args map[string]interface{}) *Row {
segments := make([][]rowSegment, 0, len(others)+1)
if len(r.segments) > 0 {
segments = append(segments, r.segments)
}
nextSegs := make([][]rowSegment, 0, len(others)+1)
toProcess := make([]*rowSegment, 0, len(others)+1)
var output []rowSegment
for _, other := range others {
if len(other.segments) > 0 {
segments = append(segments, other.segments)
}
}
for len(segments) > 0 {
shard := segments[0][0].shard
for _, segs := range segments {
if segs[0].shard < shard {
shard = segs[0].shard
}
}
nextSegs = nextSegs[:0]
toProcess := toProcess[:0]
for _, segs := range segments {
if segs[0].shard == shard {
toProcess = append(toProcess, &segs[0])
segs = segs[1:]
}
if len(segs) > 0 {
nextSegs = append(nextSegs, segs)
}
}
// at this point, "toProcess" is a list of all the segments
// sharing the lowest ID, and nextSegs is a list of all the others.
// Swap the segment lists (so we don't have to reallocate it)
segments, nextSegs = nextSegs, segments
output = append(output, *toProcess[0].GenericNaryOp(op, toProcess[1:], args))
}
return &Row{segments: output}
}
// Difference returns the diff of r and other.
func (r *Row) Difference(others ...*Row) *Row {
var output []rowSegment
@ -408,17 +348,6 @@ func (r *Row) Difference(others ...*Row) *Row {
return &Row{segments: output}
}
// GenericUnaryOp returns the results of a generic op on r.
func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row {
work := r
var segments []rowSegment
for _, segment := range work.segments {
opped := segment.GenericUnaryOp(op, args)
segments = append(segments, *opped)
}
return &Row{segments: segments}
}
// Shift returns the bitwise shift of r by n bits.
// Currently only positive shift values are supported.
//
@ -523,15 +452,6 @@ func (r *Row) Count() uint64 {
return n
}
// GenericCount applies an op to lots of things.
func (r *Row) GenericCount(op ext.BitmapOpUnaryCount, args map[string]interface{}) uint64 {
var n int64
for i := range r.segments {
n += op([]ext.Bitmap{WrapBitmap(r.segments[i].data)}, args)
}
return uint64(n)
}
// MarshalJSON returns a JSON-encoded byte slice of r.
func (r *Row) MarshalJSON() ([]byte, error) {
var o struct {
@ -644,33 +564,6 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment {
}
}
// GenericOp performs a generic op on s and other
func (s *rowSegment) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *rowSegment, args map[string]interface{}) *rowSegment {
data := op([]ext.Bitmap{WrapBitmap(s.data), WrapBitmap(other.data)}, args)
return &rowSegment{
data: UnwrapBitmap(data),
shard: s.shard,
n: data.Count(),
}
}
// GenericOp performs a generic op on s and others
func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSegment, args map[string]interface{}) *rowSegment {
bitmaps := make([]ext.Bitmap, len(others)+1)
bitmaps[0] = WrapBitmap(s.data)
for i, seg := range others {
bitmaps[i+1] = WrapBitmap(seg.data)
}
data := op(bitmaps, args)
return &rowSegment{
data: UnwrapBitmap(data),
shard: s.shard,
n: data.Count(),
}
}
// Difference returns the diff of s and other.
func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment {
datas := make([]*roaring.Bitmap, len(others))
@ -713,17 +606,6 @@ func (s *rowSegment) Shift() (*rowSegment, error) {
}, nil
}
// GenericUnaryOp returns s subject to op.
func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment {
data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args))
return &rowSegment{
data: data,
shard: s.shard,
n: data.Count(),
}
}
// SetBit sets the i-th column of the row.
func (s *rowSegment) SetBit(i uint64) (changed bool) {
s.ensureWritable()

86
server.go
View file

@ -27,13 +27,11 @@ import (
"sync"
"time"
"github.com/molecula/ext"
uuid "github.com/satori/go.uuid"
// extensions pulls in some extensions depending on build tags
_ "github.com/pilosa/pilosa/v2/extensions"
"github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
"github.com/pkg/errors"
@ -63,7 +61,6 @@ type Server struct { // nolint: maligned
hosts []string
clusterDisabled bool
serializer Serializer
extensions []*ext.ExtensionInfo
// External
systemInfo SystemInfo
@ -430,30 +427,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.cluster.confirmDownRetries = s.confirmDownRetries
s.cluster.confirmDownSleep = s.confirmDownSleep
s.holder.broadcaster = s
err = s.loadAllExtensions()
if err != nil {
s.logger.Printf("not all plugins loaded successfully")
}
if len(s.extensions) > 0 {
s.logger.Printf("loaded extensions:")
for _, ext := range s.extensions {
if ext == nil {
s.logger.Printf(" inexplicably, a nil extension?!?")
continue
}
s.logger.Printf(" %s %s: %s", ext.Name, ext.Version, ext.Description)
if ext.License != "" {
s.logger.Printf(" License: %s", ext.License)
}
if len(ext.BitmapOps) > 0 {
opList := make([]string, len(ext.BitmapOps))
for i := range ext.BitmapOps {
opList[i] = ext.BitmapOps[i].Name
}
s.logger.Printf(" Ops: %s", strings.Join(opList, ", "))
}
}
}
err = s.cluster.setup()
if err != nil {
@ -467,56 +440,6 @@ func (s *Server) InternalClient() InternalClient {
return s.defaultClient
}
// loadNewExtensions loads extensions that have been
// registered since the last call to loadNewExtensions.
func (s *Server) loadNewExtensions() error { //nolint:unused
return s.loadExtensions(ext.NewExtensions())
}
// loadAllExtensions loads all extensions.
func (s *Server) loadAllExtensions() error {
return s.loadExtensions(ext.AllExtensions())
}
func (s *Server) loadExtensions(exts []*ext.ExtensionInfo) error {
var lastError error
for _, extension := range exts {
if err := s.loadExtension(extension); err != nil {
lastError = err
}
}
return lastError
}
func (s *Server) loadExtension(extInfo *ext.ExtensionInfo) error {
if extInfo.ExtensionAPI != "v0" {
return fmt.Errorf("%s: unsupported extension API %s", extInfo.Name, extInfo.ExtensionAPI)
}
s.extensions = append(s.extensions, extInfo)
bitmapOps := extInfo.BitmapOps
bmOps, countOps, fieldOps, unknownOps := 0, 0, 0, 0
for i := range bitmapOps {
typ := bitmapOps[i].Func.BitmapOpType()
switch {
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
countOps++
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
bmOps++
case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
fieldOps++
default:
unknownOps++
}
}
err := s.executor.registerOps(bitmapOps)
if err != nil {
s.logger.Printf("warning: extension registration failed: %v", err)
} else {
pql.RegisterPluginFuncs(bitmapOps)
}
return nil
}
// UpAndDown brings the server up minimally and shuts it down
// again; basically, it exists for testing holder open and close.
func (s *Server) UpAndDown() error {
@ -545,8 +468,12 @@ func (s *Server) UpAndDown() error {
func (s *Server) Open() error {
s.logger.Printf("open server")
// Start background monitoring.
s.snapshotQueue = newSnapshotQueue(10, 2, s.logger)
if s.holder.NeedsSnapshot() {
// Start background monitoring.
s.snapshotQueue = newSnapshotQueue(10, 2, s.logger)
} else {
s.snapshotQueue = defaultSnapshotQueue //TODO (twg) rethink this
}
// Log startup
err := s.holder.logStartup()
@ -711,6 +638,7 @@ func (s *Server) monitorAntiEntropy() {
// the cluster sets its state to resizing and *then* sends to
// abortAntiEntropyCh before starting to resize
}
// Sync holders.
s.logger.Printf("holder sync beginning")
s.cluster.muAntiEntropy.Lock()

12
test/index.go
View file

@ -32,7 +32,9 @@ func newIndex() *Index {
if err != nil {
panic(err)
}
index, err := pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i")
h := pilosa.NewHolder(pilosa.DefaultPartitionN)
h.Path = path
index, err := h.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
panic(err)
}
@ -42,7 +44,7 @@ func newIndex() *Index {
// MustOpenIndex returns a new, opened index at a temporary path. Panic on error.
func MustOpenIndex() *Index {
index := newIndex()
if err := index.Open(); err != nil {
if err := index.Open(false); err != nil {
panic(err)
}
return index
@ -62,12 +64,14 @@ func (i *Index) Reopen() error {
}
path, name := i.Path(), i.Name()
i.Index, err = pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, name)
h := pilosa.NewHolder(pilosa.DefaultPartitionN)
h.Path = h.HolderPathFromIndexPath(path, name)
i.Index, err = h.CreateIndex(name, pilosa.IndexOptions{})
if err != nil {
return err
}
if err := i.Open(); err != nil {
if err := i.Open(false); err != nil {
return err
}
return nil

213
tx.go
View file

@ -15,7 +15,12 @@
package pilosa
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"sync"
"github.com/pilosa/pilosa/v2/roaring"
@ -50,6 +55,10 @@ const writable = true
// that have not been committed.
type Tx interface {
// Type returns "roaring", "rbf", "badger", "badger_roaring", or one of the other
// blue-green Tx types at the top of txfactory.go
Type() string
// Rollback must be called the end of read-only transactions. Either
// Rollback or Commit must be called at the end of writable transactions.
// It is safe to call Rollback multiple times, but it must be
@ -97,6 +106,11 @@ type Tx interface {
// Calling Next() on the returned roaring.ContainerIterator gives
// you a roaring.Container that is either run, array, or raw bitmap.
// Return value 'found' is true when the ckey container was present.
// ckey of 0 gives all containers (in the fragment).
//
// ContainerIterator must not have side-effects. blueGreenTx will
// call it at the very beginning of commit to verify db contents.
//
ContainerIterator(index, field, view string, shard uint64, ckey uint64) (citer roaring.ContainerIterator, found bool, err error)
// RoaringBitmap retreives the roaring.Bitmap for the entire shard.
@ -162,10 +176,52 @@ type Tx interface {
OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error)
// ImportRoaringBits does efficient bulk import using rit, a roaring.RoaringIterator.
//
// See the roaring package for details of the RoaringIterator.
//
// If clear is true, the bits from rit are cleared, otherwise they are set in the
// specifed fragment.
ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error)
//
// The data argument can be nil, its ignored for RBF/BadgerTx. It is supplied to
// RoaringTx.ImportRoaringBits() in fragment.go fragment.fillFragmentFromArchive()
// to do the traditional fragment.readStorageFromArchive() which
// does some in memory field/view/fragment metadata updates.
// It makes blueGreenTx testing viable too.
//
// ImportRoaringBits return values changed and rowSet may be inaccurate if
// the data []byte is supplied (the RoaringTx implementation neglects this for speed).
ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error)
RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error)
// SliceOfShards returns all of the shards for the specified index, field, view triple.
// Use within pilosa supposes a new read-only transaction was created just
// for the SliceOfShards() call. The original Roaring version is the only
// one that needs optionalViewPath; any other Tx implementation can ignore that.
SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error)
}
// TxStore has operations that will create and commit multiple
// Tx on a backing store.
type TxStore interface {
// DeleteFragment deletes all the containers in a fragment.
//
// This is not in a Tx because it will often do too many deletes for a single
// transaction, and clients would be suprised to find their Tx had already
// been commited and they are getting an error on double-Commit.
// Instead each TxStore implementation creates and commits as many
// transactions as needed.
//
// Argument frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
// If not nil, it must be of type *fragment. If frag is supplied, then
// index must be equal to frag.index, field equal to frag.field, view equal
// to frag.view, and shard equal to frag.shard.
//
DeleteFragment(index, field, view string, shard uint64, frag interface{}) error
// Close shuts down the database.
Close() error
}
// RawRoaringData used by ImportRoaringBits.
@ -207,10 +263,26 @@ func NewMultiTxWithIndex(writable bool, index *Index) *MultiTx {
var _ Tx = (*MultiTx)(nil)
func (mtx *MultiTx) Type() string {
return RoaringTxn
}
func (mtx *MultiTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
tx, err := mtx.txNoShard(index)
panicOn(err)
return tx.SliceOfShards(index, field, view, optionalViewPath)
}
func (mtx *MultiTx) UseRowCache() bool {
return true
}
func (mtx *MultiTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
}
func (mtx *MultiTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
tx, err := mtx.tx(index, shard)
panicOn(err)
@ -226,8 +298,10 @@ func (mtx *MultiTx) Pointer() string {
return fmt.Sprintf("%p", mtx)
}
func (tx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
panic("not done")
func (mtx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
}
func (mtx *MultiTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
@ -412,6 +486,20 @@ func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) {
return tx, nil
}
// version of the above for SliceOfShards(), where we don't have a shard.
func (mtx *MultiTx) txNoShard(index string) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
// Lookup transaction from cache.
for _, tx := range mtx.txs {
if tx.(*RoaringTx).Index.name == index {
return tx, nil
}
}
panic(fmt.Sprintf("no prior RoaringTx available, looking up index='%v'", index))
}
type multiTxKey struct {
index string
shard uint64
@ -426,10 +514,46 @@ type RoaringTx struct {
fragment *fragment
}
func (mtx *RoaringTx) Type() string {
return RoaringTxn
}
func (tx *RoaringTx) UseRowCache() bool {
return true
}
func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
// SliceOfShards is based on view.openFragments()
file, err := os.Open(filepath.Join(optionalViewPath, "fragments"))
if os.IsNotExist(err) {
return
} else if err != nil {
return nil, errors.Wrap(err, "opening fragments directory")
}
defer file.Close()
fis, err := file.Readdir(0)
if err != nil {
return nil, errors.Wrap(err, "reading fragments directory")
}
for _, fi := range fis {
if fi.IsDir() {
continue
}
// Parse filename into integer.
shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
//v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name())
continue
}
sliceOfShards = append(sliceOfShards, shard)
}
return
}
func (tx *RoaringTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
@ -442,13 +566,25 @@ func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roa
return b.Iterator()
}
func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
b, err := tx.bitmap(index, field, view, shard)
panicOn(err)
// ImportRoaringBits return values changed and rowSet will be inaccurate if
// the data []byte is supplied. This mimics the traditional roaring-per-file
// and should be faster.
func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
f, err := tx.getFragment(index, field, view, shard)
if err != nil {
return 0, nil, err
}
return b.ImportRoaringRawIterator(rit, clear, true, rowSize)
if len(data) > 0 {
// changed and rowSet are ignored anyway when len(data) > 0;
// when we are called from fragment.fillFragmentFromArchive()
// which is the only place the data []byte is supplied.
// blueGreenTx also turns off the checks in this case.
return 0, nil, f.readStorageFromArchive(bytes.NewBuffer(data))
}
changed, rowSet, err = f.storage.ImportRoaringRawIterator(rit, clear, true, rowSize)
return
}
func (tx *RoaringTx) Readonly() bool {
@ -625,8 +761,16 @@ func (tx *RoaringTx) OffsetRange(index, field, view string, shard uint64, offset
// getFragment is used by IncrementOpN() and by bitmap()
func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*fragment, error) {
// If a fragment is attached, always use it.
// If a fragment is attached, always use it. Since it was set at Tx creation,
// it is highly likely to be correct.
if tx.fragment != nil {
// but still a basic sanity check.
if tx.fragment.index != index ||
tx.fragment.field != field ||
tx.fragment.view != view ||
tx.fragment.shard != shard {
panic(fmt.Sprintf("different fragment cached vs requested. tx.fragment='%#v', index='%v', field='%v'; view='%v'; shard='%v'", tx.fragment, index, field, view, shard))
}
return tx.fragment, nil
}
@ -659,8 +803,10 @@ func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*frag
}
frag := v.Fragment(shard)
if frag == nil {
panic(fmt.Sprintf("fragment not found: %q / %q / %d", field, view, shard))
return nil, fmt.Errorf("fragment not found: %q / %q / %d", field, view, shard)
//panic(fmt.Sprintf("fragment not found: %q / %q / %d", field, view, shard))
}
// Note: we cannot cache frag into tx.fragment.
@ -677,3 +823,52 @@ func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.B
}
return frag.storage, nil
}
type RoaringStore struct{}
func NewRoaringStore() *RoaringStore {
return &RoaringStore{}
}
func (db *RoaringStore) Close() error {
return nil
}
// frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
fragment, ok := frag.(*fragment)
if !ok {
return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag)
}
// Close data files before deletion.
if err := fragment.Close(); err != nil {
return errors.Wrap(err, "closing fragment")
}
// Delete fragment file.
if err := os.Remove(fragment.path); err != nil {
return errors.Wrap(err, "deleting fragment file")
}
// Delete fragment cache file.
if err := os.Remove(fragment.cachePath()); err != nil {
return errors.Wrap(err, fmt.Sprintf("no cache file to delete for shard %d", fragment.shard))
}
return nil
}
func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
file, err := os.Open(fragmentPathForRoaring) // open the fragment file
if err != nil {
return nil, -1, err
}
fi, err := file.Stat()
if err != nil {
return nil, -1, errors.Wrap(err, "statting")
}
sz = fi.Size()
r = file
return
}

187
txfactory.go
View file

@ -22,6 +22,7 @@ import (
"strings"
"syscall"
"github.com/pilosa/pilosa/v2/rbf"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
@ -55,15 +56,34 @@ var sep = string(os.PathSeparator)
type TxFactory struct {
typeOfTx txtype
bw *BadgerDBWrapper
badgerDB *BadgerDBWrapper
rbfDB *rbf.DB
roaringDB *RoaringStore
// could have more than one *Index, but for now keep it simple,
// and allow blueGreenTx to report badger contents via idx
idx *Index
// TODO: put RBF database handle here.
}
/* want glue-green to multiplex, so don't do this directly
// but rather f.CloseStore()
func (f *TxFactory) Store() TxStore {
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return &RoaringStore{}
case badgerTxn:
return f.badgerDB
case rbfTxn:
return f.rbfDB
// case blueGreenBadgerRoaring:
// case blueGreenRoaringBadger:
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
*/
// integer types for fast switch{}
type txtype int
@ -85,6 +105,32 @@ const (
blueGreenRBFBadger txtype = 9
)
func (txf *TxFactory) NeedsSnapshot() bool {
switch txf.typeOfTx {
case noneTxn:
panic("noneTxn should not occur")
case roaringFragmentFilesTxn:
return true
case badgerTxn:
return false
case rbfTxn:
return false
case blueGreenBadgerRoaring:
return true
case blueGreenRoaringBadger:
return true
case blueGreenRBFRoaring:
return true
case blueGreenRoaringRBF:
return true
case blueGreenBadgerRBF:
return false
case blueGreenRBFBadger:
return false
}
panic(fmt.Sprintf("unknown typeOfTx '%v'", txf.typeOfTx))
}
func MustTxsrcToTxtype(txsrc string) txtype {
switch txsrc {
case RoaringTxn: // "roaring"
@ -109,29 +155,52 @@ func MustTxsrcToTxtype(txsrc string) txtype {
panic(fmt.Sprintf("unknown txsrc '%v'", txsrc))
}
func newTxFactory(txsrc string, path string) (f *TxFactory, err error) {
// always store files in a subdir of dir. If we are having one
// database or many can depend on name.
func NewTxFactory(txsrc string, dir, name string) (f *TxFactory, err error) {
ty := MustTxsrcToTxtype(txsrc)
if ty < 1 || ty > 9 {
panic(fmt.Sprintf("invalid txtype '%v'", int(ty)))
}
var bw *BadgerDBWrapper
if ty == badgerTxn || ty == 4 || ty == 5 || ty == 8 || ty == 9 {
bw, err = openBadgerDBWrapper(path)
f = &TxFactory{
typeOfTx: ty,
roaringDB: NewRoaringStore(),
}
switch ty {
case badgerTxn, blueGreenBadgerRoaring, blueGreenRoaringBadger, blueGreenBadgerRBF, blueGreenRBFBadger:
// one, big, bad-ass badger for all data: the honeyBadger.
//
// Note that having a single Tx backing store for all indexes
// enables cross-index Tx, which are important and are tested for.
path := dir + sep + "honeyBadger"
f.badgerDB, err = globalBadgerReg.openBadgerDBWrapper(path)
// TODO(jea): figure out what the appropriate error path is here.
//fmt.Printf("warning: could not open badgerdb on path '%v': '%v'. For safety, we are opening a new '%v-fallback' instead\n", path, err, path+"-fallback")
if err != nil {
//bw, err = newBadgerDBWrapper(path + "-fallback")
bw, err = newBadgerDBWrapper(path)
f.badgerDB, err = globalBadgerReg.newBadgerDBWrapper(path)
}
panicOn(err)
bw.doAllocZero = true
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot open badger db. path='%v'", path))
}
// electric-fence like finding of access to mmapped data beyond
// transaction end time.
f.badgerDB.doAllocZero = true
}
return &TxFactory{
typeOfTx: ty,
bw: bw,
}, err
switch ty {
case rbfTxn, blueGreenRBFRoaring, blueGreenRoaringRBF, blueGreenBadgerRBF, blueGreenRBFBadger:
path := dir + sep + name + ".rbf"
f.rbfDB = rbf.NewDB(path)
if err := f.rbfDB.Open(); err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot open rbf db. path='%v'", path))
}
}
return f, err
}
// Txo holds the transaction options
@ -153,34 +222,35 @@ func (f *TxFactory) DeleteIndex(name string) error {
// from holder.go:955, by default is already done there with os.RemoveAll()
return nil
case badgerTxn:
return f.bw.DeleteIndex(name)
return f.badgerDB.DeleteIndex(name)
case rbfTxn:
panic("todo rbfTxn DeleteIndex(name)")
case blueGreenBadgerRoaring:
return f.bw.DeleteIndex(name)
return f.badgerDB.DeleteIndex(name)
case blueGreenRoaringBadger:
return f.bw.DeleteIndex(name)
return f.badgerDB.DeleteIndex(name)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
func (f *TxFactory) Close() error {
func (f *TxFactory) DeleteFragmentFromStore(index, field, view string, shard uint64, frag *fragment) error {
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return nil
return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
case badgerTxn:
// note cannot actually close Badger here.
// causes problems b/c tries holder.DeleteIndex tries to delete the index after db is closed.
//return f.bw.Close()
return nil
return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
case rbfTxn:
panic("todo rbfTxn Close()")
//return f.rbfDB.DeleteFragment(index, field, view, shard, frag)
return nil
case blueGreenBadgerRoaring:
return nil
_ = f.badgerDB.DeleteFragment(index, field, view, shard, frag)
return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenRoaringBadger:
return nil
_ = f.roaringDB.DeleteFragment(index, field, view, shard, frag)
return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
func (f *TxFactory) CloseIndex(idx *Index) error {
@ -188,10 +258,14 @@ func (f *TxFactory) CloseIndex(idx *Index) error {
case roaringFragmentFilesTxn:
return nil
case badgerTxn:
// note cannot actually close Badger here.
// causes problems b/c tries holder.DeleteIndex tries to delete the index after db is closed.
//return f.badgerDB.Close()
return nil
case rbfTxn:
panic("todo rbfTxn CloseIndex()")
// for same reason as above may not be able to close here.
//return f.rbfDB.Close()
return nil
case blueGreenBadgerRoaring:
return nil
case blueGreenRoaringBadger:
@ -202,23 +276,66 @@ func (f *TxFactory) CloseIndex(idx *Index) error {
func (f *TxFactory) NewTx(o Txo) Tx {
indexName := ""
if o.Index != nil {
indexName = o.Index.name
}
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
case badgerTxn:
btx := f.bw.NewBadgerTx(o.Write)
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
return btx
case rbfTxn:
panic("todo rbfTxn creation")
/*
rbftx, err := f.rbfDB.Begin(o.Write)
if err != nil {
errors.Wrap(err, "rbfDB.Begin transaction errored")
}
return rbftx
*/
case blueGreenBadgerRoaring:
btx := f.bw.NewBadgerTx(o.Write)
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(btx, rtx, f.idx)
case blueGreenRoaringBadger:
btx := f.bw.NewBadgerTx(o.Write)
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(rtx, btx, f.idx)
/*
case blueGreenBadgerRBF:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
rbftx, err := f.rbfDB.Begin(o.Write)
if err != nil {
errors.Wrap(err, "rbfDB.Begin transaction errored")
}
return newBlueGreenTx(btx, rbftx, f.idx)
case blueGreenRBFBadger:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
rbftx, err := f.rbfDB.Begin(o.Write)
if err != nil {
errors.Wrap(err, "rbfDB.Begin transaction errored")
}
return newBlueGreenTx(rbftx, btx, f.idx)
case blueGreenRBFRoaring:
rbftx, err := f.rbfDB.Begin(o.Write)
if err != nil {
errors.Wrap(err, "rbfDB.Begin transaction errored")
}
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(rbftx, rtx, f.idx)
case blueGreenRoaringRBF:
rbftx, err := f.rbfDB.Begin(o.Write)
if err != nil {
errors.Wrap(err, "rbfDB.Begin transaction errored")
}
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(rtx, rbftx, f.idx)
*/
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
@ -255,7 +372,7 @@ func (ty txtype) String() string {
// Hence to view uncommited keys, you must provide in optionalUseThisTx the
// Tx in which they have been added.
func (idx *Index) StringifiedBadgerKeys(optionalUseThisTx Tx) string {
return idx.Txf.bw.StringifiedBadgerKeys(optionalUseThisTx)
return idx.Txf.badgerDB.StringifiedBadgerKeys(optionalUseThisTx)
}
// fragmentSpecFromRoaringPath takes a path releative to the

41
utils_internal_test.go
View file

@ -15,8 +15,6 @@
package pilosa
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"path/filepath"
@ -396,9 +394,8 @@ func (b bcast) SendTo(to *Node, m Message) error {
return nil
}
// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing.
// FollowResizeInstruction is a version of cluster.followResizeInstruction used for testing.
func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error {
// Prepare the return message.
complete := &ResizeInstructionComplete{
JobID: instr.JobID,
@ -420,7 +417,8 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
}
// Sync available shards.
for _, is := range instr.NodeStatus.Indexes {
for k, is := range instr.NodeStatus.Indexes {
_ = k
for _, fs := range is.Fields {
f := destCluster.holder.Field(is.Name, fs.Name)
@ -451,23 +449,28 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
}
}
buf := bytes.NewBuffer(nil)
// this is the *test* version of a network call, transferring fragments between
// nodes in a cluster. So it is allowed to be kind of a hack.
bw := bufio.NewWriter(buf)
br := bufio.NewReader(buf)
// there will be two -badgerdb directories/databases, we need to copy
// from src to dest the fragment. This simulates sending the fragment over the network.
srcIdx := srcCluster.holder.Index(src.Index)
srctx := srcIdx.Txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment})
// Get the fragment from source.
if _, err := srcFragment.WriteTo(bw); err != nil {
return err
}
// Flush the bufio.buf to the io.Writer (buf).
bw.Flush()
// Write data to destination.
if _, err := destFragment.ReadFrom(br); err != nil {
return err
destIdx := destCluster.holder.Index(src.Index)
desttx := destIdx.Txf.NewTx(Txo{Write: writable, Index: destIdx, Fragment: destFragment})
citer, _, err := srctx.ContainerIterator(src.Index, src.Field, src.View, src.Shard, 0)
panicOn(err)
d := destFragment
for citer.Next() {
ckey, c := citer.Value()
err := desttx.PutContainer(d.index, d.field, d.view, d.shard, ckey, c)
panicOn(err)
}
citer.Close()
panicOn(desttx.Commit())
srctx.Rollback()
}
return nil

104
view.go
View file

@ -50,6 +50,7 @@ type view struct {
qualifiedName string
holder *Holder
idx *Index
fieldType string
cacheType string
@ -143,7 +144,8 @@ func (v *view) open() error {
}
v.holder.Logger.Debugf("open fragments for index/field/view: %s/%s/%s", v.index, v.field, v.name)
if err := v.openFragments(); err != nil {
if err := v.openFragmentsInTx(); err != nil {
return errors.Wrap(err, "opening fragments")
}
@ -159,64 +161,27 @@ func (v *view) open() error {
var workQueue = make(chan struct{}, runtime.NumCPU()*2)
// openFragments opens and initializes the fragments inside the view.
func (v *view) openFragments() error {
file, err := os.Open(filepath.Join(v.path, "fragments"))
if os.IsNotExist(err) {
return nil
} else if err != nil {
return errors.Wrap(err, "opening fragments directory")
}
defer file.Close()
// replaces v.openFragments() with Tx generic code.
func (v *view) openFragmentsInTx() error {
fis, err := file.Readdir(0)
tx := v.idx.Txf.NewTx(Txo{Write: !writable, Index: v.idx})
defer tx.Rollback()
shards, err := tx.SliceOfShards(v.index, v.field, v.name, v.path)
if err != nil {
return errors.Wrap(err, "reading fragments directory")
return errors.Wrap(err, "SliceOfShards")
}
eg, ctx := errgroup.WithContext(context.Background())
var mu sync.Mutex
fileLoop:
for _, loopFi := range fis {
select {
case <-ctx.Done():
break fileLoop
default:
fi := loopFi
if fi.IsDir() {
continue
}
// Parse filename into integer.
shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name())
continue
}
workQueue <- struct{}{}
v.holder.Logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard)
eg.Go(func() error {
defer func() {
<-workQueue
}()
frag := v.newFragment(v.fragmentPath(shard), shard)
if err := frag.Open(); err != nil {
return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)
}
frag.RowAttrStore = v.rowAttrStore
v.holder.Logger.Debugf("add index/field/view/fragment to view.fragments: %s/%s/%s/%d", v.index, v.field, v.name, shard)
mu.Lock()
v.fragments[frag.shard] = frag
v.addKnownShard(frag.shard)
mu.Unlock()
return nil
})
for _, shard := range shards {
frag := v.newFragment(v.fragmentPath(shard), shard)
if err := frag.Open(); err != nil {
return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)
}
frag.RowAttrStore = v.rowAttrStore
v.holder.Logger.Debugf("add index/field/view/fragment to view.fragments: %s/%s/%s/%d", v.index, v.field, v.name, shard)
v.fragments[frag.shard] = frag
v.addKnownShard(frag.shard)
}
return eg.Wait()
return nil
}
// close closes the view and its fragments.
@ -359,6 +324,16 @@ func (v *view) notifyIfNewShard(shard uint64) {
}
func (v *view) newFragment(path string, shard uint64) *fragment {
if v.holder != nil && v.idx != nil {
// A view must have its v.idx *Index registered with its holder.
// Otherwise TestField_AvailableShards crashes, as one example.
hIdx := v.holder.Index(v.idx.name)
if hIdx == nil && v.idx != nil {
v.holder.addIndexFromField(v.idx)
}
}
frag := newFragment(v.holder, path, v.index, v.field, v.name, shard, v.flags())
frag.CacheType = v.cacheType
frag.CacheSize = v.cacheSize
@ -375,28 +350,17 @@ func (v *view) newFragment(path string, shard uint64) *fragment {
func (v *view) deleteFragment(shard uint64) error {
v.mu.Lock()
defer v.mu.Unlock()
fragment := v.fragments[shard]
if fragment == nil {
f := v.fragments[shard]
if f == nil {
return ErrFragmentNotFound
}
v.holder.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard)
// Close data files before deletion.
if err := fragment.Close(); err != nil {
return errors.Wrap(err, "closing fragment")
idx := f.holder.Index(v.index)
if err := idx.Txf.DeleteFragmentFromStore(f.index, f.field, f.view, f.shard, f); err != nil {
return errors.Wrap(err, "DeleteFragment")
}
// Delete fragment file.
if err := os.Remove(fragment.path); err != nil {
return errors.Wrap(err, "deleting fragment file")
}
// Delete fragment cache file.
if err := os.Remove(fragment.cachePath()); err != nil {
v.holder.Logger.Printf("no cache file to delete for shard %d", shard)
}
delete(v.fragments, shard)
v.removeKnownShard(shard)

10
view_internal_test.go
View file

@ -34,7 +34,15 @@ func mustOpenView(index, field, name string) *view {
CacheSize: DefaultCacheSize,
}
v := newView(NewHolder(DefaultPartitionN), path, index, field, name, fo)
h := NewHolder(DefaultPartitionN)
h.Path = path
// h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment
idx, err := h.createIndex(index, IndexOptions{})
_ = idx
panicOn(err)
v := newView(h, path, index, field, name, fo)
v.idx = idx
if err := v.open(); err != nil {
panic(err)
}

30
vprint.go
View file

@ -111,3 +111,33 @@ func FileLine(depth int) string {
func stack() string {
return string(debug.Stack())
}
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}