scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-05 08:10:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into improve-tx-error-messages

Browse source
This commit is contained in:
alanbernstein 2020-08-03 12:34:22 -05:00 • committed by GitHub
commit 27edff60b0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
61 changed files with 5566 additions and 1371 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
.circleci/config.yml
View file

@ -55,6 +55,13 @@ jobs:
- checkout-plus
- run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/local/bin v1.23.8
- run: make golangci-lint
go-mod-tidy:
executor:
name: golang
steps:
- checkout-plus
- run: go mod tidy
- run: git diff --exit-code -- go.mod go.sum
test-build-arm:
executor:
name: golang
@ -168,6 +175,9 @@ workflows:
- check-license-headers:
requires:
- setup
- go-mod-tidy:
requires:
- setup
- test-build-arm:
requires:
- setup

56
Makefile
View file

@ -16,16 +16,13 @@ RELEASE_ENABLED = $(subst 0,,$(RELEASE))
NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" )
BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
BUILD_TAGS += shardwidth$(SHARD_WIDTH)
BUILD_TAGS += $(foreach p,$(PLUGINS),plugin$(p))
define LICENSE_HASH_CODE
head -13 $1 | sed -e 's/Copyright 20[0-9][0-9]/Copyright 20XX/g' | shasum | cut -f 1 -d " "
endef
LICENSE_HASH=$(shell $(call LICENSE_HASH_CODE, pilosa.go))
PLUGINS=distinct
export GO111MODULE=on
export GOPRIVATE=github.com/molecula
export PLUGINS
# Run tests and compile Pilosa
default: test build
@ -97,7 +94,7 @@ clustertests: vendor
# Like clustertests, but rebuilds all images.
clustertests-build: vendor
docker-compose -f $(DOCKER_COMPOSE) down
docker-compose -f $(DOCKER_COMPOSE) down -v
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
# Create prerelease builds
@ -152,25 +149,76 @@ docker-test:
# run top tests, not subdirs. print summary red/green after.
# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
topt:
mv log.topt.roar log.topt.roar.prev || true
go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.roar
@echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
@echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l
topt-badger:
mv log.topt.badger log.topt.badger.prev || true
PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger
@echo " log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l
@echo " log.topt.badger red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l
topt-badger-race:
mv log.topt.badger-race log.topt.badger-race.prev || true
PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger-race
@echo " log.topt.badger-race green: \c"; cat log.topt.badger-race | grep PASS |wc -l
@echo " log.topt.badger-race red: \c"; cat log.topt.badger-race | grep '\-\-\- FAIL' |wc -l
topt-rbf:
mv log.topt.rbf log.topt.rbf.prev || true
PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.rbf
@echo " log.topt.rbf green: \c"; cat log.topt.rbf | grep PASS |wc -l
@echo " log.topt.rbf red: \c"; cat log.topt.rbf | grep '\-\-\- FAIL' |wc -l
topt-race:
mv log.topt.race log.topt.race.prev || true
go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.race
@echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
@echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l
# blue-green checks. These run two different storage engines (rbf, roaring, or badger)
# and compare each transaction for a result.
bg-rr: # shorthand for bluegreen test with A:badger; B:roaring
mv log.bg-rr log.bg-rr.prev || true
set -o pipefail; PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rr
@echo " log.bg-rr green: \c"; cat log.bg-rr | grep PASS |wc -l
@echo " log.bg-rr red: \c"; cat log.bg-rr | grep '\-\-\- FAIL' |wc -l
rr-bg: # bluegreen with A:roaring; B:badger (B's values are returned).
mv log.bg.roar_bg log.bg.roar_bg.prev || true
set -o pipefail; PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-bg
##PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-bg
@echo " log.rr-bg green: \c"; cat log.rr-bg | grep PASS |wc -l
@echo " log.rr-bg red: \c"; cat log.rr-bg | grep '\-\-\- FAIL' |wc -l
rbf-rr:
mv log.rbf-rr log.rbf-rr.prev || true
set -o pipefail; PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-rr
@echo " log.rbf-rr green: \c"; cat log.rbf-rr | grep PASS |wc -l
@echo " log.rbf-rr red: \c"; cat log.rbf-rr | grep '\-\-\- FAIL' |wc -l
rr-rbf:
mv log.rr-rbf log.rr-rbf.prev || true
set -o pipefail; PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-rbf
@echo " log.rr-rbf green: \c"; cat log.rr-rbf | grep PASS |wc -l
@echo " log.rr-rbf red: \c"; cat log.rr-rbf | grep '\-\-\- FAIL' |wc -l
rbf-bg:
mv log.rbf-bg log.rbf-bg.prev || true
set -o pipefail; PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-bg
@echo " log.rbf-bg green: \c"; cat log.rbf-bg | grep PASS |wc -l
@echo " log.rbf-bg red: \c"; cat log.rbf-bg | grep '\-\-\- FAIL' |wc -l
bg-rbf:
mv log.bg-rbf log.bg-rbf.prev || true
set -o pipefail; PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rbf
@echo " log.bg-rbf green: \c"; cat log.bg-rbf | grep PASS |wc -l
@echo " log.bg-rbf red: \c"; cat log.bg-rbf | grep '\-\-\- FAIL' |wc -l
# Run golangci-lint
golangci-lint: require-golangci-lint
golangci-lint run --skip-files '.*\.peg\.go'

9
api.go
View file

@ -371,16 +371,9 @@ func importWorker(importWork chan importJob) {
var doClear bool
switch doAction {
case RequestActionOverwrite:
// TODO(jea): the question here is, why are we commiting this separately from j.tx?
// why doesn't j.tx suffice? It doesn't but why/which is correct?
tx := j.field.holder.indexes[j.field.index].Txf.NewTx(Txo{Write: true, Field: j.field})
defer tx.Rollback()
if err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block); err != nil {
if err := j.field.importRoaringOverwrite(j.ctx, j.tx, viewData, j.shard, viewName, j.req.Block); err != nil {
return errors.Wrap(err, "importing roaring as overwrite")
}
if err := tx.Commit(); err != nil {
return errors.Wrap(err, "commit of importing roaring as overwrite")
}
case RequestActionClear:
doClear = true
fallthrough

4
api_test.go
View file

@ -165,6 +165,8 @@ func TestAPI_ImportColumnAttrs(t *testing.T) {
}
func TestAPI_Import(t *testing.T) {
skipForRBF(t)
c := test.MustRunCluster(t, 2,
[]server.CommandOption{
server.OptCommandServerOptions(
@ -276,6 +278,8 @@ func TestAPI_Import(t *testing.T) {
}
func TestAPI_ImportValue(t *testing.T) {
skipForRBF(t)
c := test.MustRunCluster(t, 2,
[]server.CommandOption{
server.OptCommandServerOptions(

516
badger.go
View file

@ -15,8 +15,9 @@
package pilosa
import (
"errors"
"bytes"
"fmt"
"io"
"io/ioutil"
"log"
"os"
@ -29,7 +30,9 @@ import (
"unsafe"
badger "github.com/dgraph-io/badger/v2"
badgeroptions "github.com/dgraph-io/badger/v2/options"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
// TODO: is there a more optimal time to do badger garbage collection?
@ -189,15 +192,70 @@ func (l *BadgerLog) Debugf(f string, v ...interface{}) {
l.Printf("DEBUG: "+f, v...)
}
// badgerRegistrar facilitates shutdown
// of all the badger databases started under
// tests. Its needed because most tests don't cleanup
// the *Index(es) they create. But we still
// want to shutdown badgerDB goroutines
// after tests run.
//
// It also allows opening the same path twice to
// result in sharing the same open database handle, and
// thus the same transactional guarantees.
//
type badgerRegistrar struct {
mu sync.Mutex
mp map[*BadgerDBWrapper]bool
path2db map[string]*BadgerDBWrapper
}
var globalBadgerReg *badgerRegistrar = newBadgerTestRegistrar()
func newBadgerTestRegistrar() *badgerRegistrar {
return &badgerRegistrar{
mp: make(map[*BadgerDBWrapper]bool),
path2db: make(map[string]*BadgerDBWrapper),
}
}
// register each badger created under tests, so we
// can clean them up. This is called by openBadgerDBWrapper() while
// holding the r.mu.Lock, since it needs to atomically
// check the registry and make a new instance only
// if one does not exist for its path, and otherwise
// return the existing instance.
func (r *badgerRegistrar) unprotectedRegister(w *BadgerDBWrapper) {
r.mp[w] = true
r.path2db[w.path] = w
}
// unregister removes w from r
func (r *badgerRegistrar) unregister(w *BadgerDBWrapper) {
r.mu.Lock()
delete(r.mp, w)
delete(r.path2db, w.path)
r.mu.Unlock()
}
func DumpAllBadger() {
globalBadgerReg.mu.Lock()
defer globalBadgerReg.mu.Unlock()
for w := range globalBadgerReg.mp {
_ = w
AlwaysPrintf("this badger path='%v' has: \n%v\n", w.path, w.StringifiedBadgerKeys(nil))
}
}
// newBadgerDBWrapper creates a new empty database, blowing away
// any prior path + "-badgerdb" directory.
func newBadgerDBWrapper(path string) (*BadgerDBWrapper, error) {
func (r *badgerRegistrar) newBadgerDBWrapper(path string) (*BadgerDBWrapper, error) {
bpath := badgerPath(path)
err := os.RemoveAll(bpath)
if err != nil {
return nil, err
}
return openBadgerDBWrapper(bpath)
return r.openBadgerDBWrapper(bpath)
}
// badgerPath is a helper for determining the full directory
@ -212,17 +270,46 @@ func badgerPath(path string) string {
// openBadgerDB opens the database in the bpath directoy
// without deleting any prior content. Any BadgerDB
// database directory will have the "-badgerdb" suffix.
func openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) {
//
// openBadgerDB will check the registry and make a new instance only
// if one does not exist for its bpath. Otherwise it returns
// the existing instance. This insures only one badgerDB
// per bpath in this pilosa node.
func (r *badgerRegistrar) openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) {
// now that newTxFactory can call us directly, we might not
// have the -badgerdb suffix.
if !strings.HasSuffix(bpath, "-badgerdb") {
bpath += "-badgerdb"
}
r.mu.Lock()
defer r.mu.Unlock()
w, ok := r.path2db[bpath]
if ok {
// creates the effect of having only one badger open per pilosa node.
return w, nil
}
// otherwise, make a new badger and store it in globalBadgerReg
// regular: works on amd64, but 386 doesn't work.
opt := badger.DefaultOptions(bpath).WithLogger(badgerDefaultLogger)
opt.Compression = badgeroptions.None // turn off compression.
opt.ZSTDCompressionLevel = 0 // really, just in case.
opt.SyncWrites = true // default is true, safe.
// MaxCacheSize docs:
//
// how much data cache should hold in memory. A small size of
// cache means lower memory consumption and lookups/iterations
// would take longer. It is recommended to use a cache if you're
// using compression or encryption. If compression and
// encryption both are disabled, adding a cache will lead to
// unnecessary overhead which will affect the read performance.
// Setting size to zero disables the cache altogether.
opt.MaxCacheSize = 0
opt.LoadBloomsOnOpen = false // should speed up start-up time.
// to get memory only do:
//opt := badger.DefaultOptions("").WithLogger(badgerDefaultLogger).WithInMemory(true)
@ -231,12 +318,16 @@ func openBadgerDBWrapper(bpath string) (*BadgerDBWrapper, error) {
return nil, err
}
halt := make(chan bool)
w := &BadgerDBWrapper{
w = &BadgerDBWrapper{
reg: r,
path: bpath,
db: db,
halt: halt,
hasher: NewBlake3Hasher(),
}
r.unprotectedRegister(w)
w.startStack = stack()
w.startBadgerGarbageCollectionBackgroundGoro()
return w, nil
}
@ -251,77 +342,8 @@ func (w *BadgerDBWrapper) DeleteIndex(indexName string) error {
if strings.Contains(indexName, "'") {
return fmt.Errorf("error: bad indexName `%v` in BadgerDBWrapper.DeleteIndex() call: indexName cannot contain apostrophes/single quotes.", indexName)
}
w.muDb.Lock()
defer w.muDb.Unlock()
// a) do key-ony iteration, no value fetch;
//
// b) do deletes in large batches, to avoid alot of txn overhead;
// per recommendation https://github.com/dgraph-io/badger/issues/598
//
// c) we do not, at present, try to maintain one large
// transaction with all the keys in a index in it. Because
// there can be too many keys. Hence the index will disappear
// in chucks of 100K keys, not atomically-all-at-once.
prefix := badgerIndexOnlyPrefix(indexName)
noMoreKeysWithPrefix := false
const maxDeletesPerTxn = 100000
for !noMoreKeysWithPrefix {
err := w.db.Update(func(txn *badger.Txn) error {
o := badger.DefaultIteratorOptions
o.AllVersions = false
o.PrefetchValues = false // key-only iteration, no values.
// note: panic: Unclosed iterator at time of Txn.Discard ? panic on segfault here?
// This means we messed up and Closed() the Database already; too early.
it := txn.NewIterator(o)
defer it.Close()
n := 0
goners := make([][]byte, 0, maxDeletesPerTxn)
for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
// KeyCopy() is required; Key() means corruption and possible segfault.
key := it.Item().KeyCopy(nil)
goners = append(goners, key)
n++
if n >= maxDeletesPerTxn {
break
}
}
if !it.ValidForPrefix(prefix) {
noMoreKeysWithPrefix = true // done with the full delete of up to maxDeletesPerTxn
}
for _, key := range goners {
if err := txn.Delete(key); err != nil {
return err
}
}
return nil // auto-commit happens
})
// err back from Update can be ErrConflict in case of
// a conflict. Badger docs: "Depending on the state
// of your application, you have the option to
// retry the operation if you receive this error."
panicOn(err)
} // end for: proceed to next bath of 100K keys
// Finally, run a garbage collection to delete values from the value log.
//
// "Only one GC is allowed at a time. If another value log GC
// is running, or DB has been closed, this would return an ErrRejected."
// -- https://godoc.org/github.com/dgraph-io/badger#DB.RunValueLogGC
// Still, we don't see a mutex inside the RunValueLogGC code, so
// lock muGC just to be sure.
w.muGC.Lock()
defer w.muGC.Unlock()
_ = w.db.RunValueLogGC(0.5)
return nil
return w.DeletePrefix(prefix)
}
// startBadgerGarbageCollectionBackgroundGoro handles Badger DB
@ -367,6 +389,9 @@ type BadgerDBWrapper struct {
path string
db *badger.DB
// track our registrar for Close / goro leak reporting purposes.
reg *badgerRegistrar
// openTx and openIt are BadgerDBWrapper scoped tables of all open
// transactions and iterators. These are primarily for debugging purposes.
// openTx and openIt should only be read/written after locking the muOpenTxIt mutex.
@ -404,6 +429,12 @@ type BadgerDBWrapper struct {
// safety because otherwise TestAPI_ImportColumnAttrs sees
// corrupted data.
doAllocZero bool
// stack() from our creation point, to track tests
// that haven't closed us.
startStack string
DeleteEmptyContainer bool
}
// unprotectedListOpenTxAsString is a debugging helper.
@ -437,24 +468,31 @@ func (w *BadgerDBWrapper) UnprotectedListOpenItAsString() (r string) {
// Read-only queries should set write to false, to allow more concurrency.
// Methods on a BadgerTx are thread-safe, and can be called from
// different goroutines.
func (w *BadgerDBWrapper) NewBadgerTx(write bool) (tx *BadgerTx) {
//
// initialIndexName is optional. It is set by the TxFactory from the Txo
// options provided at the Tx creation point. It allows us to recognize
// and isolate cross-index queries more quickly. It can always be empty ""
// but when set is highly useful for debugging. It has no impact
// on transaction behavior.
//
func (w *BadgerDBWrapper) NewBadgerTx(write bool, initialIndexName string) (tx *BadgerTx) {
w.muDb.Lock()
defer w.muDb.Unlock()
tx = &BadgerTx{
write: write,
tx: w.db.NewTransaction(write),
Db: w,
initloc: stack(),
doAllocZero: w.doAllocZero,
write: write,
tx: w.db.NewTransaction(write),
Db: w,
initloc: stack(),
doAllocZero: w.doAllocZero,
initialIndexName: initialIndexName,
DeleteEmptyContainer: w.DeleteEmptyContainer,
}
//vv("NewBadgerTx(write=%v) top, p=%p", write, tx)
//pp("NewBadgerTx(write=%v) top, p=%p, stack=\n\n'%v'", write, tx, stack())
if w.openTx == nil {
w.openTx = make(map[*BadgerTx]bool)
}
//pp("NewBadgerTx(write=%v); p=%p; (currently open txn: '%v', its: '%v'). initloc:'%v'", write, tx, w.unprotectedListOpenTxAsString(), w.UnprotectedListOpenItAsString(), tx.initloc)
w.muOpenTxIt.Lock()
w.openTx[tx] = write
w.muOpenTxIt.Unlock()
@ -466,6 +504,7 @@ func (w *BadgerDBWrapper) Close() (err error) {
w.muDb.Lock()
defer w.muDb.Unlock()
if !w.closed {
w.reg.unregister(w)
close(w.halt)
w.closed = true
}
@ -501,8 +540,17 @@ type BadgerTx struct {
// for tracking txn boundary issues, track all the memory
// that we deploy for roaring containers, and zero it on
// transaction commit/rollback.
acMu sync.Mutex // protect ourAllocs and ourContainers
ourAllocs [][]byte
ourContainers []*roaring.Container
initialIndexName string
DeleteEmptyContainer bool
}
func (tx *BadgerTx) Type() string {
return BadgerTxn
}
func (tx *BadgerTx) UseRowCache() bool {
@ -521,6 +569,8 @@ func (tx *BadgerTx) UseRowCache() bool {
// to transaction commit.
func (tx *BadgerTx) overWriteOurAllocs() {
tx.acMu.Lock()
defer tx.acMu.Unlock()
for _, s := range tx.ourAllocs {
// The Go compiler recognizes the following pattern and inserts
@ -529,6 +579,10 @@ func (tx *BadgerTx) overWriteOurAllocs() {
// and https://codereview.appspot.com/137880043
for i := range s {
s[i] = 0
// or
// Seebs suggested we might see even more crashes :)
// but since it will be slow (no memclr), we'll leave the default 0 for now.
//s[i] = -2
}
}
// keep this around if we need to activate out-of-mmap memory access again.
@ -632,21 +686,57 @@ func badgerKey(index, field, view string, shard uint64, roaringContainerKey uint
prefix := badgerPrefix(index, field, view, shard)
ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
return append(prefix, ckey...)
bkey := append(prefix, ckey...)
MustValidateKey(bkey)
return bkey
}
var ckeyPartExpected = []byte(";ckey@")
// MustValidatekey will panic on a bad badgerKey with an informative message.
func MustValidateKey(bkey []byte) {
n := len(bkey)
if n < 56 {
panic(fmt.Sprintf("bkey too short min size is 56 but we see %v in '%v'", n, string(bkey)))
}
beforeCkey := bkey[n-26 : n-20]
if !bytes.Equal(beforeCkey, ckeyPartExpected) {
panic(fmt.Sprintf(`bkey did not have expected ";ckey@" at 26 bytes from the end of the bkey '%v'; instead had '%v'`, string(bkey), string(beforeCkey)))
}
}
func shardFromBadgerKey(bkey []byte) (shard uint64) {
MustValidateKey(bkey)
n := len(bkey)
// idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615 -> idx:'i';fld:'f';vw:'standard';shd:'1
by := bkey[:n-27]
beg := bytes.LastIndex(by, []byte("'"))
if beg == -1 {
panic(fmt.Sprintf("bad bkey='%v' did not have single quote to being shard decoding", string(bkey)))
}
parseMe := string(by[beg+1:])
shard, err := strconv.ParseUint(parseMe, 10, 64)
if err != nil {
panic(fmt.Sprintf("could not parse parseMe '%v' in strconv.ParseUint(), error: '%v'", parseMe, err))
}
return shard
}
// badgerKeyAndPrefix returns the equivalent of badgerKey() and badgerPrefix() calls.
func badgerKeyAndPrefix(index, field, view string, shard uint64, roaringContainerKey uint64) (key, prefix []byte) {
prefix = badgerPrefix(index, field, view, shard)
ckey := []byte(fmt.Sprintf("%020d", roaringContainerKey))
return append(prefix, ckey...), prefix
bkey := append(prefix, ckey...)
MustValidateKey(bkey)
return bkey, prefix
}
var _ = badgerKeyAndPrefix // keep linter happy
// badgerKeyExtractContainerKey extracts the containerKey from bkey.
func badgerKeyExtractContainerKey(bkey []byte) (containerKey uint64) {
MustValidateKey(bkey)
// The zero padding means that the container-key is always the last 20 bytes of the bkey.
//
// Be sure to catch the problematic case of a user passing in only a prefix. A prefix
@ -665,11 +755,15 @@ func badgerKeyExtractContainerKey(bkey []byte) (containerKey uint64) {
return
}
func badgerAllShardPrefix(index, field, view string) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:", index, field, view))
}
// badgerPrefix returns everything from badgerKey up to and
// including the '@' fune in a badger key. The prefix excludes the roaring container key itself.
// NB must be kept in sync with badgerKey() and badgerKeyExtractContainerKey().
func badgerPrefix(index, field, view string, shard uint64) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%x';ckey@", index, field, view, shard))
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';vw:'%v';shd:'%020v';ckey@", index, field, view, shard))
}
// badgerIndexOnlyPrefix returns a prefix suitable for DeleteIndex and a key-scan to
@ -683,6 +777,11 @@ func badgerIndexOnlyPrefix(indexName string) []byte {
return []byte(fmt.Sprintf("idx:'%v';", indexName))
}
// same for deleting a whole field.
func badgerFieldPrefix(index, field string) []byte {
return []byte(fmt.Sprintf("idx:'%v';fld:'%v';", index, field))
}
// Container returns the requested roaring.Container, selected by fragment and ckey
func (tx *BadgerTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) {
@ -869,6 +968,38 @@ func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64
return exists, err
}
func (tx *BadgerTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
prefix := badgerAllShardPrefix(index, field, view)
bi := NewBadgerIterator(tx, prefix)
defer bi.Close()
bi.Seek(prefix)
if !bi.it.Valid() {
return
}
lastShard := uint64(0)
firstDone := false
for bi.Next() {
item := bi.it.Item()
key := item.Key()
shard := shardFromBadgerKey(key)
if firstDone {
if shard != lastShard {
sliceOfShards = append(sliceOfShards, shard)
}
lastShard = shard
} else {
// first time
lastShard = shard
firstDone = true
sliceOfShards = append(sliceOfShards, shard)
}
}
return
}
// key is the container key for the first roaring Container
// roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will
// return the first container at or after key. found will be true if a
@ -877,10 +1008,10 @@ func (tx *BadgerTx) Contains(index, field, view string, shard uint64, key uint64
// BadgerTx notes: We auto-stop at the end of this shard, not going beyond.
func (tx *BadgerTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
// needle example: "index:'i';field:'f';view:'v';shard:'0';key@00000000000000000000"
// needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000"
needle := badgerKey(index, field, view, shard, firstRoaringContainerKey)
// prefix example: "index:'i';field:'f';view:'v';shard:'0';key@"
// prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@"
prefix := badgerPrefix(index, field, view, shard)
bi := NewBadgerIterator(tx, prefix)
@ -892,7 +1023,10 @@ func (tx *BadgerTx) ContainerIterator(index, field, view string, shard uint64, f
if !bi.it.ValidForPrefix(prefix) {
return bi, false, nil
}
return bi, true, nil
item := bi.it.Item()
// have to compare b/c badger might give us valid iterator
// that is past our needle if needle isn't present.
return bi, bytes.Equal(item.Key(), needle), nil
}
// BadgerIterator is the iterator returned from a BadgerTx.ContainerIterator() call.
@ -917,12 +1051,6 @@ func NewBadgerIterator(tx *BadgerTx, prefix []byte) (bi *BadgerIterator) {
tx.Db.muOpenTxIt.Lock()
defer tx.Db.muOpenTxIt.Unlock()
defer func() {
r := recover()
if r != nil {
panic(r)
}
}()
opts := badger.DefaultIteratorOptions
opts.PrefetchValues = false // else by default, pre-fetches the 1st 100 values, which would be slow.
opts.Reverse = false
@ -1125,6 +1253,7 @@ func (tx *BadgerTx) Count(index, field, view string, shard uint64) (uint64, erro
}
// Max is the maximum bit-value in your bitmap.
// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does.
func (tx *BadgerTx) Max(index, field, view string, shard uint64) (uint64, error) {
prefix := badgerPrefix(index, field, view, shard)
@ -1133,7 +1262,10 @@ func (tx *BadgerTx) Max(index, field, view string, shard uint64) (uint64, error)
it := NewBadgerReverseIterator(tx, prefix, seekto) // this iterator is still open, when we commit/discard tx.
defer it.Close()
hb, rc := it.Value()
if !it.it.Valid() {
return 0, nil
}
hb, rc := it.Value() // getting it returns invalid, as in empty iterator
lb := rc.Max()
return hb<<16 | uint64(lb), nil
@ -1183,18 +1315,22 @@ func (tx *BadgerTx) UnionInPlace(index, field, view string, shard uint64, others
}
// CountRange returns the count of hot bits in the start, end range on the fragment.
// roaring.countRange counts the number of bits set between [start, end).
func (tx *BadgerTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
if start >= end {
return 0, nil
}
skey := highbits(start)
ekey := highbits(end)
citer, found, err := tx.ContainerIterator(index, field, view, shard, skey)
_ = found
panicOn(err)
defer citer.Close() // doesn't seem to be getting called.
if !found {
return 0, nil
}
defer citer.Close()
// If range is entirely in one container then just count that range.
if skey == ekey {
citer.Next()
@ -1284,6 +1420,7 @@ func (tx *BadgerTx) OffsetRange(index, field, view string, shard, offset, start,
bkey := item.Key()
k := badgerKeyExtractContainerKey(bkey)
// >= hi1 is correct b/c endx cannot have any lowbits set.
if uint64(k) >= hi1 {
break
}
@ -1309,7 +1446,7 @@ func (tx *BadgerTx) IncrementOpN(index, field, view string, shard uint64, change
// ImportRoaringBits handles deletes by setting clear=true.
// rowSet[rowID] returns the number of bit changed on that rowID.
func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
n := itr.Len()
if n == 0 {
return
@ -1367,7 +1504,7 @@ func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, i
changed += changes
rowSet[currRow] -= changes
if newC.N() == 0 {
if tx.DeleteEmptyContainer && newC.N() == 0 {
err = tx.RemoveContainer(index, field, view, shard, itrKey)
if err != nil {
return
@ -1399,7 +1536,7 @@ func (tx *BadgerTx) ImportRoaringBits(index, field, view string, shard uint64, i
continue
}
newC := oldC.UnionInPlace(synthC)
newC := roaring.Union(oldC, synthC) // UnionInPlace was giving us crashes on overly large containers.
if roaring.ContainerType(newC) == containerBitmap {
newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it.
@ -1447,6 +1584,10 @@ const (
func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
if len(v) == 0 {
return nil
}
// For safety we copy v, since it lives in BadgerDB's memory-mapped vlog-file,
// and Badger will recycle it after tx ends with rollback or commit.
// We copy into Go runtime GC managed memory. Technically we don't need
@ -1456,11 +1597,19 @@ func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
// TODO: performance tuning might want w := v here, if we can guarantee no access to memory past the Tx lifetime.
//
// Problem is, at least some tests appear to not respect transaction boundaries...
//
// Seebs suggested this nice variation: we could use individual mmaps for these
// copies, which would be unusable in production, but workable for testing, and then unmap them,
// which would get us probable segfaults on future accesses to them.
//
w := make([]byte, len(v))
copy(w, v) // green go test -v -run TestAPI_ImportColumnAttrs
copy(w, v)
// the copy above makes green: // green go test -v -run TestAPI_ImportColumnAttrs
//w := v // if instead of append we use v directly, it causes red: go test -v -run TestAPI_ImportColumnAttrs
// register w so we can catch out-of-tx memory access
tx.acMu.Lock()
defer tx.acMu.Unlock()
tx.ourAllocs = append(tx.ourAllocs, w)
switch typ {
@ -1483,35 +1632,39 @@ func (tx *BadgerTx) toContainer(typ byte, v []byte) (r *roaring.Container) {
// fromArray16 converts to an 8KB page
func fromArray16(a []uint16) []byte {
if len(a) == 0 {
return []byte{}
}
if len(a) > 4096 {
panic(fmt.Sprintf("cannot put more than 4096 integers into an array container: %v too big", len(a)))
}
return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2]
}
// fromArray64 converts to an 8KB page
func fromArray64(a []uint64) []byte {
if len(a) == 0 {
return []byte{}
}
return (*[8192]byte)(unsafe.Pointer(&a[0]))[:8192:8192]
}
// fromInterval16 converts to 8KB page
func fromInterval16(a []roaring.Interval16) []byte {
if len(a) == 0 {
return []byte{}
}
if len(a) > 2048 {
panic(fmt.Sprintf("cannot put more than 2048 roaring.Interval16 into a container: %v too big", len(a)))
}
return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4]
}
// badgerKey method on fragment creates a query key in the
// standard format by invoking the top level badgerKey with
// the container key being highbits(rowID * ShardWidth).
//
// Commented out for now only to keep the golangci-lint happy,
// as it has no users at the moment.
//func (f *fragment) badgerKey(rowID uint64) []byte {
// hi0 := highbits(rowID * ShardWidth)
// return badgerKey(f.index, f.field, f.view, f.shard, hi0)
//}
// StringifiedBadgerKeys returns a string with all the container
// keys available in badger.
func (w *BadgerDBWrapper) StringifiedBadgerKeys(optionalUseThisTx Tx) (r string) {
if optionalUseThisTx == nil {
tx := w.NewBadgerTx(!writable)
tx := w.NewBadgerTx(!writable, "<StringifiedBadgerKeys>")
defer tx.Rollback()
r = stringifiedBadgerKeysTx(tx)
return
@ -1548,6 +1701,10 @@ func (tx *BadgerTx) countBitsSet(bkey []byte) (n int) {
return
}
func (tx *BadgerTx) Dump() {
fmt.Printf("%v\n", stringifiedBadgerKeysTx(tx))
}
// stringifiedBadgerKeysTx reports all the badger keys and a
// corresponding blake3 hash viewable by txn within the entire
// badger database.
@ -1619,6 +1776,23 @@ func asInts(a []uint64) (r []int) {
return
}
var _ = zeroKeyContainerAsString // happy linter
// for debugging
func zeroKeyContainerAsString(ct *roaring.Container) (r string) {
cts := roaring.NewSliceContainers()
cts.Put(0, ct)
rbm := &roaring.Bitmap{Containers: cts}
r = fmt.Sprintf("[%v]:", containerTypeNames[roaring.ContainerType(ct)]) + bitmapAsString(rbm)
return
}
var containerTypeNames = map[byte]string{
containerArray: "array",
containerBitmap: "bitmap",
containerRun: "run",
}
func bitmapAsString(rbm *roaring.Bitmap) (r string) {
r = "c("
slc := rbm.Slice()
@ -1699,3 +1873,107 @@ func dirAsString(path string) (r string) {
}
var _ = dirAsString // happy linter
func (w *BadgerDBWrapper) DeleteField(index, field, fieldPath string) error {
// under blue-green roaring_badger, the directory will not be found, b/c roaring will have
// already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs:
// "If the path does not exist, RemoveAll returns nil (no error)"
err := os.RemoveAll(fieldPath)
if err != nil {
return errors.Wrap(err, "removing directory")
}
prefix := badgerFieldPrefix(index, field)
return w.DeletePrefix(prefix)
}
func (w *BadgerDBWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
prefix := badgerPrefix(index, field, view, shard)
return w.DeletePrefix(prefix)
}
func (w *BadgerDBWrapper) DeletePrefix(prefix []byte) error {
w.muDb.Lock()
defer w.muDb.Unlock()
// a) do key-ony iteration, no value fetch;
//
// b) do deletes in large batches, to avoid alot of txn overhead;
// per recommendation https://github.com/dgraph-io/badger/issues/598
//
// c) we do not, at present, try to maintain one large
// transaction with all the keys in a index in it. Because
// there can be too many keys. Hence the index will disappear
// in chucks of 100K keys, not atomically-all-at-once.
noMoreKeysWithPrefix := false
const maxDeletesPerTxn = 100000
for !noMoreKeysWithPrefix {
err := w.db.Update(func(txn *badger.Txn) error {
o := badger.DefaultIteratorOptions
o.AllVersions = false
o.PrefetchValues = false // key-only iteration, no values.
// note: panic: Unclosed iterator at time of Txn.Discard ? panic on segfault here?
// This means we messed up and Closed() the Database already; too early.
it := txn.NewIterator(o)
defer it.Close()
n := 0
goners := make([][]byte, 0, maxDeletesPerTxn)
for it.Seek(prefix); it.ValidForPrefix(prefix); it.Next() {
// KeyCopy() is required; Key() means corruption and possible segfault.
key := it.Item().KeyCopy(nil)
goners = append(goners, key)
n++
if n >= maxDeletesPerTxn {
break
}
}
if !it.ValidForPrefix(prefix) {
noMoreKeysWithPrefix = true // done with the full delete of up to maxDeletesPerTxn
}
for _, key := range goners {
if err := txn.Delete(key); err != nil {
return err
}
}
return nil // auto-commit happens
})
// err back from Update can be ErrConflict in case of
// a conflict. Badger docs: "Depending on the state
// of your application, you have the option to
// retry the operation if you receive this error."
panicOn(err)
} // end for: proceed to next bath of 100K keys
// Finally, run a garbage collection to delete values from the value log.
//
// "Only one GC is allowed at a time. If another value log GC
// is running, or DB has been closed, this would return an ErrRejected."
// -- https://godoc.org/github.com/dgraph-io/badger#DB.RunValueLogGC
// Still, we don't see a mutex inside the RunValueLogGC code, so
// lock muGC just to be sure.
w.muGC.Lock()
defer w.muGC.Unlock()
_ = w.db.RunValueLogGC(0.5)
return nil
}
func (tx *BadgerTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
rbm, err := tx.RoaringBitmap(index, field, view, shard)
if err != nil {
return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap")
}
var buf bytes.Buffer
sz, err = rbm.WriteTo(&buf)
if err != nil {
return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)")
}
return ioutil.NopCloser(&buf), sz, err
}

263
badger_test.go
View file

@ -12,12 +12,18 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// explanation of build tags:
//
// badgerdb builds but won't run in 32-bit 386 world, as of 2020 July 20.
// See https://github.com/dgraph-io/badger/issues/1384 for any progress.
// What we see is that the value-log allocations immediately run out of
// memory. So we turn off 386 with a build tag to keep the .circleci happy.
//
// gendebug_test will have a TestMain if build tag generationdebug is on,
// so we avoid conflicting with that debug scenario.
// +build !386
// +build !generationdebug
package pilosa
@ -40,7 +46,7 @@ var _ = &roaring.Bitmap{}
func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) {
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
@ -53,7 +59,7 @@ func badgerDBMustHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string
func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, bitvalue uint64) {
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
@ -64,7 +70,7 @@ func badgerDBMustNotHaveBitvalue(dbwrap *BadgerDBWrapper, index, field, view str
}
func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
// add a bit
changed, err := tx.Add(index, field, view, shard, doBatched, putme)
@ -82,14 +88,14 @@ func badgerDBMustSetBitvalue(dbwrap *BadgerDBWrapper, index, field, view string,
}
func badgerDBMustDeleteBitvalueContainer(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
hi := highbits(putme)
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
panicOn(tx.Commit())
}
func badgerDBMustDeleteBitvalue(dbwrap *BadgerDBWrapper, index, field, view string, shard uint64, putme uint64) {
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
_, err := tx.Remove(index, field, view, shard, putme)
panicOn(err)
panicOn(tx.Commit())
@ -99,7 +105,7 @@ func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func()
var err error
fn := badgerPath(path)
panicOn(os.RemoveAll(fn))
w, err = newBadgerDBWrapper(path)
w, err = globalBadgerReg.newBadgerDBWrapper(path)
panicOn(err)
// verify it is empty
@ -109,6 +115,7 @@ func mustOpenEmptyBadgerWrapper(path string) (w *BadgerDBWrapper, cleaner func()
}
return w, func() {
w.Close() // stop any started background GC goroutine.
os.RemoveAll(fn)
}
}
@ -118,8 +125,8 @@ func TestBadger_SetBitmap(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SetBitmap")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
bitvalue := uint64(0)
changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
if changed <= 0 {
@ -140,7 +147,7 @@ func TestBadger_SetBitmap(t *testing.T) {
// commited, so should be visible outside the txn
//
tx2 := dbwrap.NewBadgerTx(!writable)
tx2 := dbwrap.NewBadgerTx(!writable, index)
exists, err = tx2.Contains(index, field, view, shard, bitvalue)
panicOn(err)
if !exists {
@ -159,9 +166,9 @@ func TestBadger_OffsetRange(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SetBitmap")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
bitvalue := uint64(1 << 20)
changed, err := tx.Add(index, field, view, shard, doBatched, bitvalue)
if changed <= 0 {
@ -194,7 +201,7 @@ func TestBadger_OffsetRange(t *testing.T) {
start := uint64(0 << 16)
endx := bitvalue + 1<<16
tx2 := dbwrap.NewBadgerTx(!writable)
tx2 := dbwrap.NewBadgerTx(!writable, index)
rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
tx2.Rollback()
@ -208,7 +215,7 @@ func TestBadger_OffsetRange(t *testing.T) {
// now offset by 2M
offset = uint64(2 << 20)
tx3 := dbwrap.NewBadgerTx(!writable)
tx3 := dbwrap.NewBadgerTx(!writable, index)
rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
tx3.Rollback()
@ -236,7 +243,7 @@ func TestBadger_Count_on_many_containers(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
n, err := tx.Count(index, field, view, shard)
@ -252,7 +259,7 @@ func TestBadger_Count_dense_containers(t *testing.T) {
defer dbwrap.Close()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
expected := 0
// can't do more than about 100k writes per badger txn by default, so
@ -280,9 +287,9 @@ func TestBadger_ContainerIterator_on_empty(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(!writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
bitvalue := uint64(0)
citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue)
panicOn(err)
@ -298,9 +305,9 @@ func TestBadger_ContainerIterator_on_one_bit(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
bitvalue := uint64(42)
@ -398,9 +405,9 @@ func TestBadger_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
putme := uint64(1<<16) + 3 // in the key:1 container
searchme := putme + 1
@ -453,9 +460,9 @@ func TestBadger_ContainerIterator_empty_iteration_loop(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_empty_iteration_loop")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
putme := uint64(1<<16) + 3 // in the key:1 container
searchme := uint64(1 << 17) // in the next container, key:2
@ -503,9 +510,9 @@ func TestBadger_ForEach_on_one_bit(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ContainerIterator_on_one_bit")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
bitvalue := uint64(42)
@ -563,7 +570,7 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// delete, but rollback instead of commit
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
hi := highbits(putme)
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
tx.Rollback()
@ -572,7 +579,7 @@ func TestBadger_RemoveContainer_one_bit_test(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// c) within one Tx, after delete it should be gone as viewed within the txn.
tx = dbwrap.NewBadgerTx(writable)
tx = dbwrap.NewBadgerTx(writable, index)
hi = highbits(putme)
exists, err := tx.Contains(index, field, view, shard, putme)
@ -624,7 +631,7 @@ func TestBadger_Remove_one_bit_test(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// delete, but rollback instead of commit
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
hi, lo := highbits(putme), lowbits(putme)
_, _ = hi, lo
_, err := tx.Remove(index, field, view, shard, hi)
@ -635,7 +642,7 @@ func TestBadger_Remove_one_bit_test(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
// c) within one Tx, after delete it should be gone as viewed within the txn.
tx = dbwrap.NewBadgerTx(writable)
tx = dbwrap.NewBadgerTx(writable, index)
exists, err := tx.Contains(index, field, view, shard, putme)
panicOn(err)
@ -724,7 +731,7 @@ func TestBadger_Max_on_many_containers(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
max, err := tx.Max(index, field, view, shard)
@ -742,7 +749,7 @@ func TestBadger_Min_on_many_containers(t *testing.T) {
index, field, view, shard := "i", "f", "v", uint64(0)
// verify no containers flag works
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
min, containersExist, err := tx.Min(index, field, view, shard)
_ = min
panicOn(err)
@ -759,7 +766,7 @@ func TestBadger_Min_on_many_containers(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx = dbwrap.NewBadgerTx(!writable)
tx = dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
min, containersExist, err = tx.Min(index, field, view, shard)
@ -780,7 +787,7 @@ func TestBadger_CountRange_on_many_containers(t *testing.T) {
index, field, view, shard := "i", "f", "v", uint64(0)
// verify no containers flag works
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
panicOn(err)
if n != 0 {
@ -796,7 +803,7 @@ func TestBadger_CountRange_on_many_containers(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx = dbwrap.NewBadgerTx(!writable)
tx = dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
@ -824,7 +831,7 @@ func TestBadger_CountRange_middle_container(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
// pick out just the middle container with the 1 bit set on it.
@ -849,7 +856,7 @@ func TestBadger_CountRange_many_middle_container(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
// get them all
@ -880,7 +887,7 @@ func TestBadger_UnionInPlace(t *testing.T) {
badgerDBMustHaveBitvalue(dbwrap, index, field, view, shard, putme)
}
tx2 := dbwrap.NewBadgerTx(!writable)
tx2 := dbwrap.NewBadgerTx(!writable, index)
n, err := tx2.Count(index, field, view, shard)
panicOn(err)
if n != 2 {
@ -895,7 +902,7 @@ func TestBadger_UnionInPlace(t *testing.T) {
}
mustAddR(others3.Add(4 << 16)) // outside the 2<<16 container
tx := dbwrap.NewBadgerTx(writable)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
err = tx.UnionInPlace(index, field, view, shard, others, others2, others3)
panicOn(err)
@ -920,7 +927,7 @@ func TestBadger_RoaringBitmap(t *testing.T) {
putme := expected
badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
rbm, err := tx.RoaringBitmap(index, field, view, shard)
@ -951,10 +958,8 @@ func TestBadger_reverse_badger_iterator_and_prefix_valid(t *testing.T) {
return nil
})
panicOn(err)
//vv("stringifiedBadgerKeys(db) = '%v'", stringifiedBadgerKeys(dbwrap.db))
// allkeys:["a:0", "a:1", "a:2", "b:0", "b:1", "b:2", "c:0", "c:1", "c:2", ]'
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, "no-index-avail")
prefix := []byte("b:")
it := NewBadgerIterator(tx, prefix)
@ -1014,10 +1019,8 @@ func TestBadger_just_reverse_badger_iterator_and_prefix_valid(t *testing.T) {
return nil
})
panicOn(err)
//vv("stringifiedBadgerKeys(db) = '%v'", stringifiedBadgerKeys(dbwrap.db))
// allkeys:["a:0", "a:1", "a:2", "b:0", "b:1", "b:2", "c:0", "c:1", "c:2", ]'
tx := dbwrap.NewBadgerTx(!writable)
tx := dbwrap.NewBadgerTx(!writable, "no-index-avail")
seekto := []byte("c:")
prefix := []byte("b:")
@ -1046,9 +1049,10 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
tx.DeleteEmptyContainer = true // traditional badger Tx behavior, but not Roaring.
//bitvalue := uint64(42)
@ -1062,7 +1066,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
clear := false
logme := false
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
_ = rowSet
if changed != len(bits) {
panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err))
@ -1079,7 +1083,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
// now test the union in place with the same set gives no change.
changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
_ = rowSet
if changed != 0 {
panic(fmt.Sprintf("should have not changed any bits on the second import, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err))
@ -1103,7 +1107,7 @@ func TestBadger_ImportRoaringBits(t *testing.T) {
itr, err := roaring.NewRoaringIterator(data)
panicOn(err)
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
_ = rowSet
if changed != 1 {
panic(fmt.Sprintf("should have changed 1 bit: '%v', rowSet='%#v', err='%v'", changed, rowSet, err))
@ -1128,9 +1132,9 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits_set_nonoverlapping_bits")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
// get some roaring bits, get an itr RoaringIterator from them
rowSize := uint64(0)
@ -1148,7 +1152,7 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
clear := false
logme := false
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
_ = rowSet
if changed != len(bits) {
panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err))
@ -1165,7 +1169,7 @@ func TestBadger_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
// now import the 2nd, overlapping set and set them.
changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize)
changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil)
_ = rowSet
if changed != 4 {
panic(fmt.Sprintf("should have changed 2 bits: the 1 and the 3, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err))
@ -1178,9 +1182,9 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_ImportRoaringBits_clear_nonoverlapping_bits")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
defer tx.Rollback()
// get some roaring bits, get an itr RoaringIterator from them
rowSize := uint64(0)
@ -1198,7 +1202,7 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
clear := false
logme := false
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize)
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
_ = rowSet
if changed != len(bits) {
panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err))
@ -1216,7 +1220,7 @@ func TestBadger_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
// now import the 2nd overlapping set and clear them.
clear = true
changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize)
changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil)
_ = rowSet
if changed != 2 {
panic(fmt.Sprintf("should have changed 1 bit: the 2, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err))
@ -1252,9 +1256,9 @@ func TestBadger_DeleteIndex(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
bitvalue := uint64(777)
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
bitvalue := uint64(777)
bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16}
for _, v := range bits {
changed, err := tx.Add(index, field, view, shard, doBatched, v)
@ -1290,7 +1294,7 @@ func TestBadger_DeleteIndex(t *testing.T) {
err = dbwrap.DeleteIndex(index)
panicOn(err)
tx = dbwrap.NewBadgerTx(!writable)
tx = dbwrap.NewBadgerTx(!writable, index2)
defer tx.Rollback()
exists, err = tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
@ -1314,9 +1318,9 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteIndex_over100k")
defer clean()
defer dbwrap.Close()
tx := dbwrap.NewBadgerTx(writable)
bitvalue := uint64(777)
index, field, view, shard := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
bitvalue := uint64(777)
limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction.
//limit := uint64(101)
for v := uint64(1); v < limit; v++ {
@ -1328,7 +1332,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) {
panicOn(err)
if v%100000 == 0 {
panicOn(tx.Commit())
tx = dbwrap.NewBadgerTx(writable)
tx = dbwrap.NewBadgerTx(writable, index)
}
}
@ -1345,7 +1349,7 @@ func TestBadger_DeleteIndex_over100k(t *testing.T) {
err = dbwrap.DeleteIndex(index)
panicOn(err)
tx = dbwrap.NewBadgerTx(!writable)
tx = dbwrap.NewBadgerTx(!writable, index2)
defer tx.Rollback()
exists, err := tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
@ -1445,3 +1449,134 @@ func mustAddR(changed bool, err error) {
func mustRemove(changeCount int, err error) {
panicOn(err)
}
func TestBadger_DeleteFragment(t *testing.T) {
// setup
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_DeleteFragment")
defer clean()
defer dbwrap.Close()
index, field, view, shard0 := "i", "f", "v", uint64(0)
tx := dbwrap.NewBadgerTx(writable, index)
shard1 := uint64(1)
bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16}
shards := []uint64{shard0, shard1}
for _, s := range shards {
for _, v := range bits {
changed, err := tx.Add(index, field, view, s, doBatched, v)
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
}
}
for _, s := range shards {
for _, v := range bits {
exists, err := tx.Contains(index, field, view, s, v)
panicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!!")
}
}
}
err := tx.Commit()
panicOn(err)
// end of setup
survivor := shard0
victim := shard1
err = dbwrap.DeleteFragment(index, field, view, victim, nil)
panicOn(err)
tx = dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
for _, s := range shards {
for _, v := range bits {
exists, err := tx.Contains(index, field, view, s, v)
panicOn(err)
if s == survivor {
if !exists {
panic(fmt.Sprintf("ARG survivor died : bit %v", v))
}
} else if s == victim { // victim, should have been deleted
if exists {
panic(fmt.Sprintf("ARG victim lived : bit %v", v))
}
}
}
}
}
func TestBadger_shardFromBadgerKey(t *testing.T) {
if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'1';ckey@18446744073709551615")) != 1 {
panic("problem")
}
if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'0';ckey@18446744073709551615")) != 0 {
panic("problem")
}
if shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@18446744073709551615")) != 18446744073709551615 {
panic("problem")
}
func() {
defer func() {
r := recover()
if r == nil {
panic("should have panic-ed")
}
}()
// called for the panic of a short ckey, only 19 bytes instead of 20
shardFromBadgerKey([]byte("idx:'i';fld:'f';vw:'standard';shd:'18446744073709551615';ckey@1844674407370955161"))
}()
}
func TestBadger_SliceOfShards(t *testing.T) {
dbwrap, clean := mustOpenEmptyBadgerWrapper("TestBadger_SliceOfShards")
defer clean()
defer dbwrap.Close()
index, field, view := "i", "f", "v"
shards := []uint64{0, 1, 2, 3, 1000001, 2000001}
putme := uint64(179)
for _, shard := range shards {
badgerDBMustSetBitvalue(dbwrap, index, field, view, shard, putme)
}
tx := dbwrap.NewBadgerTx(!writable, index)
defer tx.Rollback()
slc, err := tx.SliceOfShards(index, field, view, "")
panicOn(err)
for i := range shards {
if shards[i] != slc[i] {
panic(fmt.Sprintf("expected at i=%v that slc[i]=%v = shards[i]=%v", i, slc[i], shards[i]))
}
}
}
func reportTestBadgersNeedingClose() {
globalBadgerReg.mu.Lock()
defer globalBadgerReg.mu.Unlock()
n := len(globalBadgerReg.mp)
if n > 0 {
AlwaysPrintf("*** these badgers are still open (n=%v):", n)
i := 0
for w := range globalBadgerReg.mp {
AlwaysPrintf("i=%v, w p=%p stack:\n%v\n\n", i, w, w.startStack)
i++
}
}
}
var _ = reportTestBadgersNeedingClose // happy linter
func TestMain(m *testing.M) {
ret := m.Run()
//reportTestBadgersNeedingClose()
os.Exit(ret)
}

484
bluegreentx.go
View file

@ -15,28 +15,65 @@
package pilosa
import (
"bytes"
"fmt"
"io"
"reflect"
"sort"
"sync"
"github.com/pilosa/pilosa/v2/roaring"
)
// blueGreenTx runs two Tx together and notices differences in their output.
// By convention, the 'b' Tx is the output that is returned to caller.
//
// Warning: DATA RACES are expected if RoaringTx is one side of the Tx pair.
// The checkDatabase() call will do reads of the fragments at Commit/Rollback,
// while the snapshotqueue may be doing writes.
//
// Do not run with go test -race and expect it to be race free.
//
type blueGreenTx struct {
a Tx
b Tx // b's output is returned
as string
bs string
idx *Index
checker blueGreenChecker
mu sync.Mutex
rollbackOrCommitDone bool
}
func newBlueGreenTx(a, b Tx, idx *Index) *blueGreenTx {
return &blueGreenTx{a: a, b: b, idx: idx}
as := a.Type()
bs := b.Type()
return &blueGreenTx{a: a, b: b, idx: idx, as: as, bs: bs}
}
var _ = newBlueGreenTx // keep linter happy
var _ Tx = (*blueGreenTx)(nil)
func (c *blueGreenTx) Type() string {
return c.a.Type() + "_" + c.b.Type()
}
var blueGreenTxDumpMut sync.Mutex
func (c *blueGreenTx) Dump() {
blueGreenTxDumpMut.Lock()
defer blueGreenTxDumpMut.Unlock()
fmt.Printf("%v blueGreenTx.Dump ============== \n", FileLine(2))
fmt.Printf("A(%v) Dump:\n", c.as)
c.a.Dump()
fmt.Printf("B(%v) Dump:\n", c.bs)
c.b.Dump()
}
func (c *blueGreenTx) Readonly() bool {
a := c.a.Readonly()
b := c.b.Readonly()
@ -47,6 +84,8 @@ func (c *blueGreenTx) Readonly() bool {
}
func (c *blueGreenTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
c.checker.see(index, field, view, shard)
// TODO(jea): does this need to be different, to handle c.a iteration at the same time?
return c.b.NewTxIterator(index, field, view, shard)
}
@ -55,11 +94,94 @@ func (c *blueGreenTx) Pointer() string {
}
func (c *blueGreenTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
c.checker.see(index, field, view, shard)
c.a.IncrementOpN(index, field, view, shard, changedN)
c.b.IncrementOpN(index, field, view, shard, changedN)
}
// compareTxState is called for the first Commit or Rollback a blueGreenTx sees.
func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
here := fmt.Sprintf("%v/%v/%v/%v", index, field, view, shard)
aIter, aFound, aErr := c.a.ContainerIterator(index, field, view, shard, 0)
bIter, bFound, bErr := c.b.ContainerIterator(index, field, view, shard, 0)
if aErr == nil || aIter != nil {
defer aIter.Close()
}
if bErr == nil || bIter != nil {
defer bIter.Close()
}
if aFound != bFound {
c.Dump()
panic(fmt.Sprintf("compareTxState[%v]: A(%v) ContainerIterator had aFound=%v, but B(%v) had bFound=%v; at '%v'", here, c.as, aFound, c.bs, bFound, stack()))
}
if aErr != nil || bErr != nil {
if aErr != nil && bErr != nil {
c.Dump()
panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err '%v'; B(%v) reported err '%v' at %v", here, c.as, aErr, c.bs, bErr, stack()))
}
if aErr != nil {
c.Dump()
panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err %v at %v; but B(%v) did not", here, c.as, aErr, c.bs, stack()))
}
if bErr != nil {
c.Dump()
panic(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, stack()))
}
}
for aIter.Next() {
aKey, aValue := aIter.Value()
if !bIter.Next() {
c.Dump()
panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, stack()))
}
bKey, bValue := bIter.Value()
if bKey != aKey {
AlwaysPrintf("problem in caller %v", Caller(2))
c.Dump()
panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, stack())) // crashing here on TestBSIGroup_importValue
}
if err := aValue.BitwiseCompare(bValue); err != nil {
c.Dump()
panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack()))
}
}
// end checking everything in A, but does B have more?
if bIter.Next() {
bKey, _ := bIter.Value()
c.Dump()
panic(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, at %v", here, c.bs, bKey, c.as, stack()))
}
}
func (c *blueGreenTx) checkDatabase() {
c.checker.mu.Lock()
defer c.checker.mu.Unlock()
// seen() returns nil on 2nd or any further call,
// so only the first Commit() or Rollback() does this.
for index, fields := range c.checker.seen() {
for field, views := range fields {
for view, shards := range views {
for shard := range shards {
c.compareTxState(index, field, view, shard)
}
}
}
}
}
func (c *blueGreenTx) Rollback() {
c.mu.Lock()
defer c.mu.Unlock()
if c.rollbackOrCommitDone {
return
}
c.rollbackOrCommitDone = true
c.checkDatabase()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
@ -71,6 +193,13 @@ func (c *blueGreenTx) Rollback() {
}
func (c *blueGreenTx) Commit() error {
c.mu.Lock()
defer c.mu.Unlock()
if c.rollbackOrCommitDone {
return nil
}
c.rollbackOrCommitDone = true
c.checkDatabase()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
@ -86,6 +215,7 @@ func (c *blueGreenTx) Commit() error {
}
func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
@ -96,10 +226,18 @@ func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*r
_, _ = a, errA
b, errB := c.b.RoaringBitmap(index, field, view, shard)
compareErrors(errA, errB)
slcA := a.Slice()
slcB := b.Slice()
if !reflect.DeepEqual(slcA, slcB) {
panic("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!")
}
return b, errB
}
func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
@ -116,6 +254,7 @@ func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uin
}
func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
@ -126,17 +265,19 @@ func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key
errB := c.b.PutContainer(index, field, view, shard, key, rc)
compareErrors(errA, errB)
/* draft idea of how to check the full databases afterwards:
hashA := c.a.RootHashString()
hashB := c.b.RootHashString()
if hashA != hashB {
panic(fmt.Sprintf("hashA = '%v' but hashB = '%v'", hashA, hashB))
}
*/
return errB
}
func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
c.checker.see(index, field, view, shard)
// these are the first port of call for debugging, so we leave them in.
// ================== begin save comments.
//c.checkDatabase()
//vv("got past database check at TOP of ImportRoaringBits")
//c.Dump()
//vv("done with top dump; clear=%v", clear)
// ================== end save comments.
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
@ -146,50 +287,38 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64,
// remember where the iterator started, so we can replay it a second time.
rit2 := rit.Clone()
panicOn(err)
changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize, data)
changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize)
if changedA != changedB {
panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB))
}
if len(rowSetA) != len(rowSetB) {
panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB))
}
for k, va := range rowSetA {
vb, ok := rowSetB[k]
if !ok {
panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB))
if len(data) == 0 {
// okay to check! otherwise we are in the fragment.fillFragmentFromArchive
// case where we know that RoaringTx.ImportRoaringBits changed and rowSet will
// be inaccurate.
if changedA != changedB {
panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB))
}
if va != vb {
panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb))
if len(rowSetA) != len(rowSetB) {
panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB))
}
for k, va := range rowSetA {
vb, ok := rowSetB[k]
if !ok {
panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB))
}
if va != vb {
panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb))
}
}
}
compareErrors(errA, errB)
//compareDatabases(c.a, c.b)
c.checkDatabase()
return changedB, rowSetB, errB
}
/* // TODO: get a database-wide checksum working
func compareDatabases(a, b Tx) {
index, field, view, shard := "i", "f", "v", uint64(0)
ha, errA := a.WholeDatabaseBlake3Hash(index, field, view, shard)
panicOn(errA)
hb, errB := b.WholeDatabaseBlake3Hash(index, field, view, shard)
panicOn(errB)
if ha != hb {
panic(fmt.Sprintf("a.WholeDatabaseBlake3Hash(%T) = '%v' but b.WholeDatabaseBlake3Hash(%T) = '%v'", a, ha, b, hb))
}
}
*/
func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
@ -207,6 +336,7 @@ func (c *blueGreenTx) UseRowCache() bool {
}
func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Add() panic '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, stack())
@ -249,6 +379,7 @@ func compareErrors(errA, errB error) {
}
func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
@ -263,6 +394,7 @@ func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint6
}
func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
@ -278,21 +410,81 @@ func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint
}
func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
// TODO: need to return a blueGreenIterator too, that does close/next operations on both A and B.
ait, afound, errA := c.a.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
_, _, _ = ait, afound, errA
bit, bfound, errB := c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
compareErrors(errA, errB)
return bit, bfound, errB
// INVAR: errA == errB, so only need to check one.
if errB != nil {
// RoaringTx can return an iterator and an error, so be sure Close it we have it.
if ait != nil {
ait.Close()
}
if bit != nil {
bit.Close()
}
return nil, bfound, errB
}
// INVAR: errA == errB == nil
bgi := NewBlueGreenIterator(c, ait, bit)
return bgi, bfound, errB
}
func NewBlueGreenIterator(tx *blueGreenTx, ait, bit roaring.ContainerIterator) *blueGreenIterator {
return &blueGreenIterator{
tx: tx,
as: tx.as,
bs: tx.bs,
ait: ait,
bit: bit,
}
}
type blueGreenIterator struct {
tx *blueGreenTx
as string
bs string
ait roaring.ContainerIterator
bit roaring.ContainerIterator
}
func (bgi *blueGreenIterator) Next() bool {
na := bgi.ait.Next()
nb := bgi.bit.Next()
if na != nb {
panic(fmt.Sprintf("na=%v(%v) != nb(%v)=%v", na, bgi.as, bgi.bs, nb))
}
return nb
}
func (bgi *blueGreenIterator) Value() (uint64, *roaring.Container) {
ka, ca := bgi.ait.Value()
kb, cb := bgi.bit.Value()
if ka != kb {
panic(fmt.Sprintf("ka=%v != kb=%v", ka, kb))
}
err := ca.BitwiseCompare(cb)
panicOn(err)
return kb, cb
}
func (bgi *blueGreenIterator) Close() {
bgi.ait.Close()
bgi.bit.Close()
}
// ForEach is read-only on the database, and so we only pass through to B.
// Avoids the side-effects of calling fn too many times.
func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
defer func() {
if r := recover(); r != nil {
@ -300,32 +492,30 @@ func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i
panic(r)
}
}()
errA := c.a.ForEach(index, field, view, shard, fn)
_ = errA
errB := c.b.ForEach(index, field, view, shard, fn)
_ = errB
return c.b.ForEach(index, field, view, shard, fn)
compareErrors(errA, errB)
return errB
}
// ForEachRange cannot change the database, and we also can't control
// the side effects of the fn() calls. So we only pass through to B, not A.
// No checker.see() is needed as well, because we are read-only.
func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
errA := c.a.ForEachRange(index, field, view, shard, start, end, fn)
_ = errA
errB := c.b.ForEachRange(index, field, view, shard, start, end, fn)
_ = errB
compareErrors(errA, errB)
return errB
// calling fn will have side effects; can only call it the right number of times.
// so can't do this.
// errA := c.a.ForEachRange(index, field, view, shard, start, end, fn)
return c.b.ForEachRange(index, field, view, shard, start, end, fn)
}
func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
@ -342,6 +532,7 @@ func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, er
}
func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
@ -358,6 +549,7 @@ func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, erro
}
func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
@ -374,6 +566,7 @@ func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool
}
func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
@ -387,8 +580,11 @@ func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, othe
}
func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
c.checker.see(index, field, view, shard)
//vv("CountRange start=0x%x, endx=0x%x", start, end)
defer func() {
if r := recover(); r != nil {
c.Dump()
AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
panic(r)
}
@ -397,7 +593,7 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start,
b, errB := c.b.CountRange(index, field, view, shard, start, end)
if a != b {
panic(fmt.Sprintf("a = %v, but b = %v", a, b))
panic(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b))
}
compareErrors(errA, errB)
@ -405,6 +601,7 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start,
}
func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
@ -419,3 +616,174 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star
compareErrors(errA, errB)
return b, errB
}
func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
c.Dump()
AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
rcA, szA, errA := c.a.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
rcB, szB, errB := c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
compareErrors(errA, errB)
// We are seeing Roaring vs Badger size differences on
// server/ test TestClusterResize_AddNode/ContinuousShards,
// so turn off the szA vs szB checks and MutliReaderB use. But keep them if we want to
// check RBF vs Badger for byte-for-byte compatiblity (we
// suspect the ops log or optimized bitmaps are accounting for the difference).
sizeMustMatch := false
if sizeMustMatch {
if szA != szB {
panic(fmt.Sprintf("szA(%v) = %v, but szB(%v) = %v; fragmentPathForRoaring='%v'", c.as, szA, c.bs, szB, fragmentPathForRoaring))
}
return &MultiReaderB{a: rcA, b: rcB}, szB, errB
} else {
// one db won't get data if we do
//return &MultiReaderB{a: rcA, b: rcB, allowSizeVariation: true}, szB, errB
_, _ = szA, errA
rcA.Close()
return rcB, szB, errB
}
}
func (c *blueGreenTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
// doesn't change state, so we don't really need see() call here. And we don't have a single shard for it.
//c.checker.see(index, field, view, shard) // don't have shard.
defer func() {
if r := recover(); r != nil {
c.Dump()
AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
slcA, errA := c.a.SliceOfShards(index, field, view, optionalViewPath)
slcB, errB := c.b.SliceOfShards(index, field, view, optionalViewPath)
compareErrors(errA, errB)
// sort order may be different, and that's ok.
cpa := append([]uint64{}, slcA...)
cpb := append([]uint64{}, slcB...)
sort.Slice(cpa, func(i, j int) bool { return cpa[i] < cpa[j] })
sort.Slice(cpb, func(i, j int) bool { return cpb[i] < cpb[j] })
if !reflect.DeepEqual(cpa, cpb) {
// report the first difference
ma := make(map[uint64]bool)
for _, ka := range slcA {
ma[ka] = true
}
for _, kb := range slcB {
if !ma[kb] {
c.Dump()
panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! B(%v) had %v, but A(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.bs, kb, c.as, cpa, cpb))
}
delete(ma, kb)
}
if len(ma) != 0 {
for firstDifference := range ma {
panic(fmt.Sprintf("blueGreenTx SliceOfShards diference! A(%v) had %v, but B(%v) did not. cpa='%#v'; cpb='%#v'; in the SliceOfShards returned slice.", c.as, firstDifference, c.bs, cpa, cpb))
}
}
panic(fmt.Sprintf("blueGreenTx SliceOfShards diference \n slcA(%v)='%#v';\n slcB(%v)='%#v';\n", c.as, cpa, c.bs, cpb))
}
return slcB, errB
}
// MultiReaderB is returned by RoaringBitmapReader. It verifies
// that identical byte streams are read from its two members.
type MultiReaderB struct {
a io.ReadCloser
b io.ReadCloser
allowSizeVariation bool
}
// Read implements the standard io.Reader method. It panics
// if "a" and "b" have even one byte different in their reads.
func (m *MultiReaderB) Read(p []byte) (nB int, errB error) {
nB, errB = m.b.Read(p)
p2 := make([]byte, nB)
// read (and discard after comparing for equality) the exact same amount from A.
// ReadAtLeast reads from r into buf until it has read at least
// min bytes. It returns the number of bytes copied and an error
// if fewer bytes were read. The error is EOF only if no bytes
// were read. If an EOF happens after reading fewer than min bytes,
// ReadAtLeast returns ErrUnexpectedEOF. If min is greater than
// the length of buf, ReadAtLeast returns ErrShortBuffer. On
// return, n >= min if and only if err == nil. If r returns
// an error having read at least min bytes, the error is dropped.
nA, errA := io.ReadAtLeast(m.a, p2, nB)
if !m.allowSizeVariation {
if errA == io.ErrUnexpectedEOF {
panic(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA))
}
if nA != nB {
panic(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA))
}
cmp := bytes.Compare(p[:nB], p2[:nB])
if cmp != 0 {
panic(fmt.Sprintf("MultiReaderB reads p and p2 (cmp= %v) differed.", cmp)) // \np ='%v'; \np2 ='%v'", cmp, string(p[:nB]), string(p2[:nA])))
}
}
return
}
func (m *MultiReaderB) Close() error {
m.a.Close()
return m.b.Close()
}
// blueGreenChecker is used
type blueGreenChecker struct {
visited map[string]map[string]map[string]map[uint64]struct{}
// lock mu when using visited.
// otherwise concurrent map writes on TestAPI_Import/RowIDColumnKey
mu sync.Mutex
}
// see would mark a thing as seen.
func (b *blueGreenChecker) see(index, field, view string, shard uint64) {
// keep this next Printf. Useful to see the sequence of Tx operations.
//fmt.Printf("blueGreenTx.%v\n", Caller(1))
b.mu.Lock()
defer b.mu.Unlock()
if b.visited == nil {
b.visited = make(map[string]map[string]map[string]map[uint64]struct{})
}
var visitedIdx map[string]map[string]map[uint64]struct{}
var visitedField map[string]map[uint64]struct{}
var visitedView map[uint64]struct{}
if visitedIdx = b.visited[index]; visitedIdx == nil {
visitedIdx = make(map[string]map[string]map[uint64]struct{})
b.visited[index] = visitedIdx
}
if visitedField = visitedIdx[field]; visitedField == nil {
visitedField = make(map[string]map[uint64]struct{})
visitedIdx[field] = visitedField
}
if visitedView = visitedField[view]; visitedView == nil {
visitedView = make(map[uint64]struct{})
visitedField[view] = visitedView
}
visitedView[shard] = struct{}{}
}
// seen reports the things it has seen, exactly once so
// that Rollback can be called after Commit without repeating
// the check.
func (b *blueGreenChecker) seen() map[string]map[string]map[string]map[uint64]struct{} {
return b.visited
}

91
bluegreentx_test.go Normal file
View file

@ -0,0 +1,91 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"bytes"
"io"
"io/ioutil"
"testing"
cryrand "crypto/rand"
)
func TestMultiReaderB(t *testing.T) {
// MultiReaderB should read identical chunks of bytes from both its "a" and "b"
// member io.Readers, else it should panic. This should hold for
// varying sizes of inputs.
for n := 1 << 5; n < (1 << 18); n = n*2 - 13 {
src := io.LimitReader(cryrand.Reader, int64(n))
a := make([]byte, n)
nr := 0
for nr < n {
na, err := src.Read(a)
panicOn(err)
nr += na
}
if nr != n {
panic("short read")
}
b := make([]byte, n)
copy(b, a)
if !bytes.Equal(a, b) {
panic("test prep failed")
}
m := &MultiReaderB{
a: ioutil.NopCloser(bytes.NewBuffer(a)),
b: ioutil.NopCloser(bytes.NewBuffer(b)),
}
// should not trigger the internal panic of MultiReadB
ncp, err := io.Copy(ioutil.Discard, m)
panicOn(err)
if ncp != int64(n) {
panic("short copy")
}
for victim := 0; victim < n; victim += 7 {
copy(b, a)
if victim%2 == 0 {
// corrupt b
b[victim] = (b[victim] + 1) % 255
} else {
// corrupt a
a[victim] = (a[victim] + 1) % 255
}
m = &MultiReaderB{
a: ioutil.NopCloser(bytes.NewBuffer(a)),
b: ioutil.NopCloser(bytes.NewBuffer(b)),
}
helperShouldPanicOnCopy(m)
}
}
}
func helperShouldPanicOnCopy(m *MultiReaderB) {
// differences in bytes read should be noticed
defer func() {
r := recover()
if r == nil {
panic("expected panic on byte difference but didn't see it")
}
}()
_, _ = io.Copy(ioutil.Discard, m)
}

32
catcher.go
View file

@ -16,6 +16,7 @@ package pilosa
import (
"fmt"
"io"
"github.com/pilosa/pilosa/v2/roaring"
)
@ -51,14 +52,18 @@ func (c *catcherTx) WholeDatabaseBlake3Hash(index, field, view string, shard uin
return c.b.WholeDatabaseBlake3Hash(index, field, view, shard)
}
func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
}
func (c *catcherTx) Dump() {
c.b.Dump()
}
func (c *catcherTx) Readonly() bool {
@ -275,3 +280,26 @@ func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start,
}()
return c.b.OffsetRange(index, field, view, shard, offset, start, end)
}
func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
}
func (c *catcherTx) Type() string {
return c.b.Type()
}
func (c *catcherTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see SliceOfShards() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.SliceOfShards(index, field, view, optionalViewPath)
}

54
cluster.go
View file

@ -110,6 +110,17 @@ func (a Nodes) ContainsID(id string) bool {
return false
}
// NodeByID returns the node for an ID. If the ID is not found,
// it returns nil.
func (a Nodes) NodeByID(id string) *Node {
for _, n := range a {
if n.ID == id {
return n
}
}
return nil
}
// Filter returns a new list of nodes with node removed.
func (a Nodes) Filter(n *Node) []*Node {
other := make([]*Node, 0, len(a))
@ -514,7 +525,7 @@ func (c *cluster) unprotectedSetState(state string) {
cleaner.Cluster = c
cleaner.Closing = c.closing
// Clean holder.
// Clean holder. This is where the shard gets removed after resize.
if err := cleaner.CleanHolder(); err != nil {
c.logger.Printf("holder clean error: err=%s", err)
}
@ -1027,20 +1038,49 @@ func (c *cluster) ownsShard(nodeID string, index string, shard uint64) bool {
func (c *cluster) partitionNodes(partitionID int) []*Node {
// Default replica count to between one and the number of nodes.
// The replica count can be zero if there are no nodes.
// Assume that c.nodes may be missing a node that is part of the cluster but not currently present.
// The partition calculation must use the full cluster size in BOTH cases:
// - use len(c.Topology.nodeIDs) instead of len(c.nodes),
// - collect nodes from c.Topology.nodeIDs rather than from c.nodes,
// - when the node is missing, it should be considered, found absent from c.nodes, then omitted from the return slice.
// Use c.Topology to determine cluster membership when it
// exists and contains data. Otherwise, fall back to using
// c.nodes. The only time c.Topology should be nil is in
// tests.
var useTopology bool
if c.Topology != nil && len(c.Topology.nodeIDs) > 0 {
useTopology = true
}
replicaN := c.ReplicaN
if replicaN > len(c.nodes) {
replicaN = len(c.nodes)
var nodeN int
if useTopology {
nodeN = len(c.Topology.nodeIDs)
} else {
nodeN = len(c.nodes)
}
if replicaN > nodeN {
replicaN = nodeN
} else if replicaN == 0 {
replicaN = 1
}
// Determine primary owner node.
nodeIndex := c.Hasher.Hash(uint64(partitionID), len(c.nodes))
nodeIndex := c.Hasher.Hash(uint64(partitionID), nodeN)
// Collect nodes around the ring.
nodes := make([]*Node, replicaN)
nodes := make([]*Node, 0, replicaN)
for i := 0; i < replicaN; i++ {
nodes[i] = c.nodes[(nodeIndex+i)%len(c.nodes)]
if useTopology {
maybeNodeID := c.Topology.nodeIDs[(nodeIndex+i)%nodeN]
if node := Nodes(c.nodes).NodeByID(maybeNodeID); node != nil {
nodes = append(nodes, node)
}
} else {
nodes = append(nodes, c.nodes[(nodeIndex+i)%len(c.nodes)])
}
}
return nodes
@ -2193,7 +2233,7 @@ func (c *cluster) nodeStatus() *NodeStatus {
func (c *cluster) mergeClusterStatus(cs *ClusterStatus) error {
c.mu.Lock()
defer c.mu.Unlock()
c.logger.Printf("merge cluster status: node=%s cluster=%v", c.Node.ID, cs)
c.logger.Printf("merge cluster status: node=%s cluster=%v, topologySize=%v", c.Node.ID, cs, len(c.Topology.nodeIDs))
// Ignore status updates from self (coordinator).
if c.unprotectedIsCoordinator() {
return nil

14
cluster_internal_test.go
View file

@ -96,7 +96,9 @@ func newIndexWithTempPath(name string) *Index {
if err != nil {
panic(err)
}
index, err := NewIndex(NewHolder(DefaultPartitionN), path, name)
h := NewHolder(DefaultPartitionN)
h.Path = path
index, err := h.CreateIndex(name, IndexOptions{})
if err != nil {
panic(err)
}
@ -160,7 +162,7 @@ func TestFragSources(t *testing.T) {
defer idx.Close()
// Obtain transaction.
tx := &RoaringTx{Index: idx}
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx})
defer tx.Rollback()
field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
@ -809,12 +811,12 @@ func TestCluster_ResizeStates(t *testing.T) {
if idx0 == nil {
t.Fatal(`idx0 was nil, could not retrieve Index("i")`)
}
//idx0.Dump("node0")
// addNode needs to block until the resize process has completed.
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
}
node1 := tc.Clusters[1]
// Ensure that nodes come up in state NORMAL.
@ -846,15 +848,12 @@ func TestCluster_ResizeStates(t *testing.T) {
if idx1 == nil {
t.Fatal(`idx1 was nil, could not retrieve Index("i")`)
}
//idx0.Dump("after rebalance, node0")
//idx1.Dump("after rebalance, node1")
// Ensure checksums are the same.
if chksum, err := node1Fragment.Checksum(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
// badger red: TestCluster_ResizeStates/Multiple_nodes,_with_data: cluster_internal_test.go:841: expected standard view checksum to match: ef46db3751d8e999 - fad4de25ee696ca0
}
// Close TestCluster.
@ -872,6 +871,7 @@ func TestAE(t *testing.T) {
c.abortAntiEntropy()
close(ch)
}()
defer c.abortAntiEntropyQ() // avoid leaking a goroutine.
select {
case <-ch:
return
@ -883,11 +883,13 @@ func TestAE(t *testing.T) {
t.Run("AbortBlocksInitialized", func(t *testing.T) {
c := newCluster()
c.initializeAntiEntropy()
ch := make(chan struct{})
go func() {
c.abortAntiEntropy()
close(ch)
}()
defer c.abortAntiEntropyQ() // avoid leak of goroutine.
select {
case <-ch:
t.Fatalf("aborting anti entropy on an initialized cluster didn't block")

150
cmd/loader/loader.go Normal file
View file

@ -0,0 +1,150 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"archive/tar"
"compress/gzip"
"context"
"time"
//"fmt"
"fmt"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/http"
"io"
"io/ioutil"
gohttp "net/http"
//"log"
"os"
//"path/filepath"
//"sort"
"strconv"
"strings"
)
func UploadTar(srcFile string, client *http.InternalClient) error {
t0 := time.Now()
f, err := os.Open(srcFile)
if err != nil {
return (err)
}
defer f.Close()
var tarReader *tar.Reader
if strings.HasSuffix(srcFile, "gz") {
gzf, err := gzip.NewReader(f)
if err != nil {
return err
}
tarReader = tar.NewReader(gzf)
} else {
tarReader = tar.NewReader(f)
}
viewData := make(map[string][]byte)
//given ordered by index/field/view
//trait_store/product_count__commercial_cd_or_share_certificate/views/bsig_product_count__commercial_cd_or_share_certificate/fragments/255
lastIndex := ""
lastField := ""
lastShard := uint64(0)
//vv("top of tar loop")
n := 0
for {
header, err := tarReader.Next()
if err == io.EOF {
if header != nil {
panic("header should not be nil on err io.EOF")
}
//submit any stuff we have left
if len(viewData) > 0 {
request := &pilosa.ImportRoaringRequest{
Views: viewData,
}
// Submit(lastIndex, lastField, lastShard, request)
//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
uri := GetImportRoaringURI(lastIndex, lastShard)
err := client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)
panicOn(err)
//vv("done with submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
}
return nil
}
//vv("got header '%v'", header.Name)
n++
if n%500 == 0 {
vv("n = %v, progress, elapsed '%v'", n, time.Since(t0))
}
parts := strings.Split(header.Name, "/")
index := parts[0]
field := parts[1]
view := parts[3]
shard, err := strconv.ParseUint(parts[5], 10, 64)
if err != nil {
return err
}
// TODO: shards can be loaded in parallel, so maybe farm out to a worker set of goro.
if index != lastIndex || field != lastField || shard != lastShard {
if len(viewData) > 0 {
request := &pilosa.ImportRoaringRequest{
Views: viewData,
}
//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
uri := GetImportRoaringURI(lastIndex, lastShard)
panicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request))
viewData = make(map[string][]byte)
//vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0))
}
}
roaringData, err := ioutil.ReadAll(tarReader)
if err != nil {
return err
}
if _, already := viewData[view]; already {
panic(fmt.Sprintf("view '%v' already present!", view))
}
viewData[view] = roaringData
lastIndex = index
lastField = field
//lastShard = shard
//vv("bottom of loop")
}
}
func main() {
host := "127.0.0.1:10101"
h := &gohttp.Client{}
c, err := http.NewInternalClient(host, h)
panicOn(err)
tarSrcPath := "q2.tar.gz"
t0 := time.Now()
panicOn(UploadTar(tarSrcPath, c))
vv("total elapsed '%v'", time.Since(t0))
}
var globURI *pilosa.URI
func init() {
var err error
globURI, err = pilosa.NewURIFromHostPort("127.0.0.1", 10101)
panicOn(err)
}
// get correct node to go to.
func GetImportRoaringURI(index string, shard uint64) *pilosa.URI {
return globURI
}

177
cmd/loader/vprint.go Normal file
View file

@ -0,0 +1,177 @@
// home: https://github.com/glyerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main
import (
"fmt"
"io"
"os"
"path"
"runtime"
"runtime/debug"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("\n%s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}
// happy linter:
var _ = DirExists
var _ = FileExists
var _ = Caller
var _ = stack
var _ = RFC3339MsecTz0
var _ = RFC3339UsecTz0
var _ = AlwaysPrintf
var _ = FileSize

282
executor.go
View file

@ -19,12 +19,12 @@ import (
"encoding/json"
"fmt"
"math"
"math/bits"
"sort"
"strings"
"sync"
"time"
"github.com/molecula/ext"
"github.com/pilosa/pilosa/v2/pql"
pb "github.com/pilosa/pilosa/v2/proto"
"github.com/pilosa/pilosa/v2/roaring"
@ -64,12 +64,6 @@ type executor struct {
workersWG sync.WaitGroup
workerPoolSize int
work chan job
// global registry to check for name clashes
additionalOps map[string]*ext.BitmapOp
// typed registries we can use in lookups
additionalBitmapOps map[string]ext.BitmapOpBitmap
additionalCountOps map[string]ext.BitmapOpUnaryCount
additionalFieldOps map[string]ext.BitmapOpBSIBitmap
}
// executorOption is a functional option type for pilosa.Executor
@ -125,38 +119,6 @@ func (e *executor) Close() error {
return nil
}
func (e *executor) registerOps(ops []ext.BitmapOp) error {
if e.additionalOps == nil {
e.additionalOps = make(map[string]*ext.BitmapOp)
e.additionalBitmapOps = make(map[string]ext.BitmapOpBitmap)
e.additionalCountOps = make(map[string]ext.BitmapOpUnaryCount)
e.additionalFieldOps = make(map[string]ext.BitmapOpBSIBitmap)
}
for i, op := range ops {
name := op.Name
if _, exists := e.additionalOps[name]; exists {
return fmt.Errorf("op name '%s' already defined", name)
}
e.additionalOps[name] = &ops[i]
typ := ops[i].Func.BitmapOpType()
switch {
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
e.additionalCountOps[name] = ops[i].Func.(ext.BitmapOpUnaryCount)
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
e.additionalBitmapOps[name] = ops[i].Func.(ext.BitmapOpBitmap)
case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
if fn, ok := ops[i].Func.(ext.BitmapOpBSIBitmapPrecall); ok {
e.additionalFieldOps[name] = ext.BitmapOpBSIBitmap(fn)
} else {
e.additionalFieldOps[name] = ops[i].Func.(ext.BitmapOpBSIBitmap)
}
default:
return fmt.Errorf("unsupported types for '%s': input type %d, output type %d", name, typ.Input, typ.Output)
}
}
return nil
}
// Execute executes a PQL query.
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *execOptions) (QueryResponse, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.Execute")
@ -228,7 +190,6 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
} else if err := validateQueryContext(ctx); err != nil {
return resp, err
}
resp.Results = results
// Fill column attributes if requested.
@ -369,7 +330,6 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr
// handlePreCalls traverses the call tree looking for calls that need
// precomputed values. Right now, that's just Distinct.
func (e *executor) handlePreCalls(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
if c.Name == "Precomputed" {
idx := c.Args["valueidx"].(int64)
if idx >= 0 && idx < int64(len(opt.EmbeddedData)) {
@ -404,18 +364,17 @@ func (e *executor) handlePreCalls(ctx context.Context, tx Tx, index string, c *p
// like Distinct, where you can't predict output shard for a result
// from the shard being queried.
if newIndex != "" && newIndex != index {
if err := e.handlePreCallChildren(ctx, tx, index, c, shards, opt); err != nil {
return err
}
c.Type = pql.PrecallGlobal
index = newIndex
// we need to recompute shards, then
shards = nil
}
if err := e.handlePreCallChildren(ctx, tx, index, c, shards, opt); err != nil {
return err
}
// child calls already handled, no precall for this, so we're done
if c.Type == pql.PrecallNone {
// otherwise, handle the children
return e.handlePreCallChildren(ctx, tx, index, c, shards, opt)
return nil
}
// We don't try to handle sub-calls from here. I'm not 100%
// sure that's right, but I think the fact that they're happening
@ -548,6 +507,7 @@ func (e *executor) execute(ctx context.Context, tx Tx, index string, q *pql.Quer
if err != nil {
return nil, err
}
results = append(results, v)
// Some Calls can have significant data associated with them
// that gets generated during processing, such as Precomputed
@ -708,15 +668,6 @@ func (e *executor) executeCall(ctx context.Context, tx Tx, index string, c *pql.
return nil, err
}
// Special handling for mutation and top-n calls.
if op, ok := e.additionalCountOps[c.Name]; ok {
statFn()
return e.executeGenericCount(ctx, tx, index, c, op, shards, opt)
}
if op, ok := e.additionalFieldOps[c.Name]; ok {
statFn()
return e.executeGenericField(ctx, tx, index, c, op, shards, opt)
}
switch c.Name {
case "Sum":
statFn()
@ -739,6 +690,9 @@ func (e *executor) executeCall(ctx context.Context, tx Tx, index string, c *pql.
case "ClearRow":
statFn()
return e.executeClearRow(ctx, tx, index, c, shards, opt)
case "Distinct":
statFn()
return e.executeDistinct(ctx, tx, index, c, shards, opt)
case "Store":
statFn()
return e.executeSetRow(ctx, tx, index, c, shards, opt)
@ -1151,11 +1105,9 @@ func (e *executor) executeSum(ctx context.Context, tx Tx, index string, c *pql.C
return other, nil
}
// executeGenericField executes a generic call on a field. Note that in this
// implementation, the operation is always a BSI op.
func (e *executor) executeGenericField(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shards []uint64, opt *execOptions) (SignedRow, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericField")
span.LogKV("name", c.Name)
// executeDistinct executes a Distinct call on a field.
func (e *executor) executeDistinct(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (SignedRow, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinct")
defer span.Finish()
field := c.Args["field"]
@ -1165,7 +1117,7 @@ func (e *executor) executeGenericField(ctx context.Context, tx Tx, index string,
// Execute calls in bulk on each remote node and merge.
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
return e.executeGenericFieldShard(ctx, tx, index, c, op, shard)
return e.executeDistinctShard(ctx, tx, index, c, shard)
}
// Merge returned results at coordinating node.
@ -1440,13 +1392,6 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index stri
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeBitmapCallShard")
defer span.Finish()
if _, ok := e.additionalCountOps[c.Name]; ok {
return nil, fmt.Errorf("count op %s used as bitmap call", c.Name)
}
if op, ok := e.additionalBitmapOps[c.Name]; ok {
return e.executeGenericBitmapShard(ctx, tx, index, c, op, shard)
}
switch c.Name {
case "Row", "Range":
return e.executeRowShard(ctx, tx, index, c, shard)
@ -1464,6 +1409,8 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index stri
return e.executeShiftShard(ctx, tx, index, c, shard)
case "All": // Allow a shard computation to use All() (note, limit/offset not applied)
return e.executeAllCallShard(ctx, tx, index, c, shard)
case "Distinct":
return nil, errors.New("Distinct shouldn't be hit as a bitmap call")
case "Precomputed":
return e.executePrecomputedCallShard(ctx, tx, index, c, shard)
default:
@ -1471,11 +1418,10 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, tx Tx, index stri
}
}
// executeGenericFieldShard executes a generic/extension command on a
// single shard. Note that in this implementation, the op is always
// a BSI op.
func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBSIBitmap, shard uint64) (SignedRow, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericShard")
// executeDistinctShard executes a Distinct call on a single shard, yielding
// a SignedRow of the values found.
func (e *executor) executeDistinctShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (result SignedRow, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinctShard")
defer span.Finish()
var filter *Row
@ -1483,7 +1429,7 @@ func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index st
if len(c.Children) == 1 {
row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard)
if err != nil {
return SignedRow{}, errors.Wrap(err, "executing bitmap call")
return result, errors.Wrap(err, "executing bitmap call")
}
filter = row
if filter != nil && len(filter.segments) > 0 {
@ -1497,29 +1443,115 @@ func (e *executor) executeGenericFieldShard(ctx context.Context, tx Tx, index st
field := e.Holder.Field(index, fieldName)
if field == nil {
return SignedRow{}, nil
return result, nil
}
bsig := field.bsiGroup(fieldName)
if bsig == nil {
return SignedRow{}, nil
return result, nil
}
view := viewBSIGroupPrefix + fieldName
depth := uint64(bsig.BitDepth)
offset := bsig.Base
existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*0, ShardWidth*1)
if err != nil {
return result, err
}
if filter != nil {
existsBitmap = existsBitmap.Intersect(filterBitmap)
}
if !existsBitmap.Any() {
return result, nil
}
fragment := e.Holder.fragment(index, fieldName, viewBSIGroupPrefix+fieldName, shard)
if fragment == nil {
return SignedRow{}, nil
signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*1, ShardWidth*2)
if err != nil {
return result, nil
}
var out ext.SignedBitmap
if filterBitmap != nil {
out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{WrapBitmap(filterBitmap)}, c.Args)
} else {
out = op(ext.BitmapBSI{FieldData: WrapBitmap(fragment.storage), ShardWidth: ShardWidth, Offset: bsig.Base, Depth: bsig.BitDepth}, []ext.Bitmap{}, c.Args)
dataBitmaps := make([]*roaring.Bitmap, depth)
for i := uint64(0); i < depth; i++ {
dataBitmaps[i], err = tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*(i+2), ShardWidth*(i+3))
if err != nil {
return result, err
}
}
// we need spaces for sign bit, existence/filter bit, and data
// row bits, which we'll be grabbing 65k bits at a time
stashWords := make([]uint64, 1024*(depth+2))
bitStashes := make([][]uint64, depth)
for i := uint64(0); i < depth; i++ {
start := i * 1024
last := start + 1024
bitStashes[i] = stashWords[start:last]
i++
}
stashOffset := depth * 1024
existStash := stashWords[stashOffset : stashOffset+1024]
signStash := stashWords[stashOffset+1024 : stashOffset+2048]
dataBits := make([][]uint64, depth)
posValues := make([]uint64, 0, 64)
negValues := make([]uint64, 0, 64)
posBitmap := roaring.NewFileBitmap()
negBitmap := roaring.NewFileBitmap()
existIterator, _ := existsBitmap.Containers.Iterator(0)
for existIterator.Next() {
key, value := existIterator.Value()
if value.N() == 0 {
continue
}
exists := value.AsBitmap(existStash)
sign := signBitmap.Containers.Get(key).AsBitmap(signStash)
for i := uint64(0); i < depth; i++ {
dataBits[i] = dataBitmaps[i].Containers.Get(key).AsBitmap(bitStashes[i])
}
for idx, word := range exists {
// mask holds a mask we can test the other words against.
mask := uint64(1)
for word != 0 {
shift := uint(bits.TrailingZeros64(word))
// we shift one *more* than that, to move the
// actual one bit off.
word >>= shift + 1
mask <<= shift
value := int64(0)
for b := uint64(0); b < depth; b++ {
if dataBits[b][idx]&mask != 0 {
value += (1 << b)
}
}
if sign[idx]&mask != 0 {
value *= -1
}
value += int64(offset)
if value < 0 {
negValues = append(negValues, uint64(-value))
} else {
posValues = append(posValues, uint64(value))
}
// and now we processed that bit, so we move the mask over one.
mask <<= 1
}
if len(negValues) > 0 {
_, _ = negBitmap.AddN(negValues...)
negValues = negValues[:0]
}
if len(posValues) > 0 {
_, _ = posBitmap.AddN(posValues...)
posValues = posValues[:0]
}
}
}
return SignedRow{
Neg: NewRowFromBitmap(UnwrapBitmap(out.Neg)),
Pos: NewRowFromBitmap(UnwrapBitmap(out.Pos)),
Neg: NewRowFromBitmap(negBitmap),
Pos: NewRowFromBitmap(posBitmap),
}, nil
}
@ -2681,11 +2713,13 @@ func (e *executor) executeRowsShard(ctx context.Context, tx Tx, index string, fi
}
func (e *executor) executeRowShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowShard")
defer span.Finish()
// Handle bsiGroup ranges differently.
if c.HasConditionArg() {
// looks the same on badger/roaring. we think.
return e.executeRowBSIGroupShard(ctx, tx, index, c, shard)
}
@ -2795,6 +2829,7 @@ func (e *executor) executeRowShard(ctx context.Context, tx Tx, index string, c *
// executeRowBSIGroupShard executes a range(bsiGroup) call for a local shard.
func (e *executor) executeRowBSIGroupShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (_ *Row, err error) {
span, _ := tracing.StartSpanFromContext(ctx, "Executor.executeRowBSIGroupShard")
defer span.Finish()
@ -2975,44 +3010,6 @@ func (e *executor) executeIntersectShard(ctx context.Context, tx Tx, index strin
return other, nil
}
// executeGenericBitmapShard executes a generic bitmap call for a local shard.
func (e *executor) executeGenericBitmapShard(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpBitmap, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericBitmapShard")
defer span.Finish()
if op.BitmapOpArity() == ext.OpArityUnary {
if len(c.Children) != 1 {
return nil, fmt.Errorf("%s needs exactly one row parameter", c.Name)
}
row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard)
if err != nil {
return nil, err
}
return row.GenericUnaryOp(op.BitmapOpFunc(), c.Args), nil
}
var err error
rows := make([]*Row, len(c.Children))
for i, input := range c.Children {
rows[i], err = e.executeBitmapCallShard(ctx, tx, index, input, shard)
if err != nil {
return nil, err
}
}
var other *Row
switch op.BitmapOpArity() {
case ext.OpArityBinary:
other = rows[0]
for _, row := range rows[1:] {
other = other.GenericBinaryOp(op.BitmapOpFunc(), row, c.Args)
}
case ext.OpArityNary:
other = rows[0].GenericNaryOp(op.BitmapOpFunc(), rows[1:], c.Args)
}
other.invalidateCount()
return other, nil
}
// executeUnionShard executes a union() call for a local shard.
func (e *executor) executeUnionShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64) (*Row, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard")
@ -3164,41 +3161,6 @@ func (e *executor) executeShiftShard(ctx context.Context, tx Tx, index string, c
return row.Shift(n)
}
// executeGeneric executes a provided count-like call.
func (e *executor) executeGenericCount(ctx context.Context, tx Tx, index string, c *pql.Call, op ext.BitmapOpUnaryCount, shards []uint64, opt *execOptions) (uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeGenericCount")
defer span.Finish()
if len(c.Children) == 0 {
return 0, fmt.Errorf("%s() requires an input bitmap", c.Name)
} else if len(c.Children) > 1 {
return 0, fmt.Errorf("%s() only accepts a single bitmap input", c.Name)
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(ctx context.Context, shard uint64) (interface{}, error) {
row, err := e.executeBitmapCallShard(ctx, tx, index, c.Children[0], shard)
if err != nil {
return 0, err
}
return row.GenericCount(op, c.Args), nil
}
// Merge returned results at coordinating node.
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(uint64)
return other + v.(uint64)
}
result, err := e.mapReduce(ctx, index, shards, c, opt, mapFn, reduceFn)
if err != nil {
return 0, err
}
n, _ := result.(uint64)
return n, nil
}
// executeCount executes a count() call.
func (e *executor) executeCount(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) (uint64, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeCount")

76
executor_test.go
View file

@ -23,6 +23,7 @@ import (
"io/ioutil"
"math"
"math/rand"
"os"
"reflect"
"strconv"
"strings"
@ -513,6 +514,12 @@ func TestExecutor_Execute_Count(t *testing.T) {
}
func roaringOnlyTest(t *testing.T) {
if os.Getenv("PILOSA_TXSRC") != "roaring" {
t.Skip("skip for everything but roaring")
}
}
// Ensure a set query can be executed.
func TestExecutor_Execute_Set(t *testing.T) {
t.Run("RowIDColumnID", func(t *testing.T) {
@ -521,7 +528,7 @@ func TestExecutor_Execute_Set(t *testing.T) {
cmd := cluster[0]
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
hldr.SetBit("i", "f", 1, 0)
hldr.SetBit("i", "f", 1, 0) // creates and commits a Tx internally.
t.Run("OK", func(t *testing.T) {
hldr.ClearBit("i", "f", 11, 1)
@ -582,10 +589,10 @@ func TestExecutor_Execute_Set(t *testing.T) {
cmd := cluster[0]
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{Keys: true})
t.Run("OK", func(t *testing.T) {
hldr.SetBit("i", "f", 1, 0)
hldr.SetBit("i", "f", 1, 0) // creates and Commits a Tx internally.
if n := hldr.Row("i", "f", 11).Count(); n != 0 {
t.Fatalf("unexpected row count: %d", n)
}
@ -619,14 +626,16 @@ func TestExecutor_Execute_Set(t *testing.T) {
})
t.Run("ErrInvalidColValueType", func(t *testing.T) {
if err := index.DeleteField("f"); err != nil {
t.Fatal(err)
}
if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
if err := idx.DeleteField("f"); err != nil {
t.Fatal(err)
}
if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(2.1, f=1)`}); err == nil || strings.Contains(err.Error(), `column value must be a string or non-negative integer`) {
if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(2.1, f=1)`}); err == nil || !strings.Contains(err.Error(), "parse error") {
t.Fatal(err)
}
@ -637,9 +646,9 @@ func TestExecutor_Execute_Set(t *testing.T) {
}
})
t.Run("ErrInvalidRowValueType", func(t *testing.T) {
index := hldr.MustCreateIndexIfNotExists("inokey", pilosa.IndexOptions{})
if _, err := index.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
t.Run("ErrInvalidRowValueType", func(t *testing.T) { // // failing under badger_roaring
idx := hldr.MustCreateIndexIfNotExists("inokey", pilosa.IndexOptions{})
if _, err := idx.CreateField("f", pilosa.OptFieldTypeDefault(), pilosa.OptFieldKeys()); err != nil {
t.Fatal(err)
}
if _, err := cmd.API.Query(context.Background(), &pilosa.QueryRequest{Index: "inokey", Query: `Set(2, f=1.2)`}); err == nil || !strings.Contains(err.Error(), "row value must be a string or non-negative integer") {
@ -3413,6 +3422,7 @@ func TestExecutor_Execute_Existence(t *testing.T) {
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
@ -3438,6 +3448,7 @@ func TestExecutor_Execute_Existence(t *testing.T) {
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) {
t.Fatalf("unexpected columns after Not: %+v", bits)
}
// Reopen cluster to ensure existence field is reloaded.
if err := c[0].Reopen(); err != nil {
t.Fatal(err)
@ -3622,6 +3633,8 @@ func TestExecutor_Execute_FieldValue(t *testing.T) {
// Ensure an all query can be executed.
func TestExecutor_Execute_All(t *testing.T) {
skipForRBF(t)
t.Run("ColumnID", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
@ -4024,7 +4037,6 @@ func TestExecutor_Execute_ClearRow(t *testing.T) {
// Ensure a row can be set.
func TestExecutor_Execute_SetRow(t *testing.T) {
t.Run("Set_NewRow", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
@ -4084,8 +4096,8 @@ func TestExecutor_Execute_SetRow(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
_, err := index.CreateField("f", pilosa.OptFieldTypeDefault())
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
_, err := idx.CreateField("f", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
@ -4174,25 +4186,6 @@ func TestExecutor_Execute_SetRow(t *testing.T) {
t.Fatalf("unexpected columns: %+v", bits)
}
})
t.Run("Err_Store(Distinct)", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{TrackExistence: true})
f1, err := index.CreateField("f1", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
f2, err := index.CreateField("f2", pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatal(err)
}
q := fmt.Sprintf(`Store(Distinct(field=%s), %s=2)`, f1.Name(), f2.Name())
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: index.Name(), Query: q}); err == nil {
t.Fatalf("expected 'unsupported result type' error, got: %+v", res)
}
})
}
func benchmarkExistence(nn bool, b *testing.B) {
@ -4402,6 +4395,8 @@ func TestExecutor_Execute_Query_Error(t *testing.T) {
}
func TestExecutor_GroupByStrings(t *testing.T) {
skipForRBF(t)
c := test.MustRunCluster(t, 1)
defer c.Close()
c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "generals", pilosa.OptFieldKeys())
@ -4858,6 +4853,8 @@ func sameStringSlice(x, y []string) bool {
}
func TestExecutor_Execute_GroupBy(t *testing.T) {
skipForRBF(t)
groupByTest := func(t *testing.T, clusterSize int) {
c := test.MustRunCluster(t, 1)
defer c.Close()
@ -5312,6 +5309,7 @@ func runCallTest(t *testing.T, writeQuery string, readQueries []string, indexOpt
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
index := hldr.MustCreateIndexIfNotExists("i", *indexOptions)
defer index.Close()
_, err := index.CreateField("f", fieldOption...)
if err != nil {
t.Fatal(err)
@ -5723,12 +5721,15 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) {
if err := json.NewDecoder(bytes.NewReader(data)).Decode(schema); err != nil {
t.Fatal(err)
}
if err := api.ApplySchema(context.TODO(), schema, false); err != nil {
t.Fatal(err)
}
// AntitodePoint == row 1 b/c keys field.
writeQuery := `Set(100, type=AntidotePoint)Set(100, equip_id=100)Set(100, site_id=100)Set(100, id=100)`
for _, i := range schema.Indexes {
for k, i := range schema.Indexes {
_ = k
if _, err := api.Query(context.TODO(), &pilosa.QueryRequest{Index: i.Name, Query: writeQuery}); err != nil {
t.Fatal(err)
}
@ -5739,11 +5740,11 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) {
Intersect(
Distinct(
Intersect(Row(type=AntidotePoint)),
index=power_ts, field=equip_id),
index=equipment, field=equip_id),
Distinct(
Intersect(Row(type=AntidotePoint)),
index=power_ts, field=equip_id)
), index=equipment, field=site_id)`
index=sites, field=equip_id)
), index=power_ts, field=site_id)`
// Check if test query gives correct results (one column 100)
t.Run("Distinct", func(t *testing.T) {
@ -5760,6 +5761,7 @@ func TestExecutor_Execute_CountDistinct(t *testing.T) {
}
if r.Pos.Count() != 1 {
t.Fatalf("invalid pilosa.SignedRow.Pos.Count, expected: 1, got: %v", r.Pos.Count())
}
if r.Pos.Columns()[0] != 100 {
t.Fatalf("invalid pilosa.SignedRow.Pos.Columns, expected: [100], got: %v", r.Pos.Columns())

96
extension.go
View file

@ -1,96 +0,0 @@
// Copyright 2019 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"github.com/molecula/ext"
"github.com/pilosa/pilosa/v2/roaring"
)
// WrapBitmap yields an extension-Bitmap from a roaring Bitmap.
func WrapBitmap(bm *roaring.Bitmap) ext.Bitmap {
return wrappedBitmap{bm}
}
// wrappedBitmap is a very shallow glue shim to convert a roaring Bitmap to
// an extension Bitmap.
type wrappedBitmap struct{ *roaring.Bitmap }
// UnwrapBitmap converts an extension-bitmap to its underlying roaring Bitmap.
func UnwrapBitmap(bm ext.Bitmap) *roaring.Bitmap {
if inner, ok := bm.(wrappedBitmap); ok {
if inner.Bitmap != nil {
return inner.Bitmap
}
return roaring.NewFileBitmap()
}
return roaring.NewFileBitmap()
}
func (b wrappedBitmap) Intersect(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Intersect(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) Union(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Union(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) IntersectionCount(other ext.Bitmap) uint64 {
return b.Bitmap.IntersectionCount(other.(wrappedBitmap).Bitmap)
}
func (b wrappedBitmap) Difference(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Difference(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) Xor(other ext.Bitmap) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Xor(other.(wrappedBitmap).Bitmap)}
}
func (b wrappedBitmap) Shift(n int) (ext.Bitmap, error) {
shifted, err := b.Bitmap.Shift(n)
return wrappedBitmap{shifted}, err
}
func (b wrappedBitmap) Flip(start, last uint64) ext.Bitmap {
return wrappedBitmap{b.Bitmap.Flip(start, last)}
}
func (b wrappedBitmap) New() ext.Bitmap {
return WrapBitmap(roaring.NewFileBitmap())
}
// ContainerBits tries to get one container's worth of bits.
func (b wrappedBitmap) ContainerBits(offset uint64, target []uint64) (out []uint64) {
// it's an error to call this with a non-container-aligned offset
if offset&0xFFFF != 0 {
return nil
}
if b.Bitmap == nil {
fmt.Printf("ContainerBits on bitmap with no contents\n")
return nil
}
if b.Bitmap.Containers == nil {
fmt.Printf("ContainerBits on bitmap with nil Containers\n")
return nil
}
c := b.Bitmap.Containers.Get(offset >> 16)
if c == nil {
return nil
}
return c.AsBitmap(target)
}

20
field.go
View file

@ -92,6 +92,8 @@ type Field struct {
name string
qualifiedName string
idx *Index
viewMap map[string]*view
// Row attribute storage and cache
@ -1181,6 +1183,7 @@ func (f *Field) createViewIfNotExistsBase(name string) (*view, bool, error) {
func (f *Field) newView(path, name string) *view {
view := newView(f.holder, path, f.index, f.name, name, f.options)
view.idx = f.idx
view.rowAttrStore = f.rowAttrStore
view.stats = f.Stats
view.broadcaster = f.broadcaster
@ -1432,6 +1435,14 @@ func (f *Field) SetValue(tx Tx, columnID uint64, value int64) (changed bool, err
if err != nil {
return false, errors.Wrap(err, "creating view")
}
if view.holder == nil {
panic("view.holder should not be nil")
}
if view.idx == nil {
panic("view.idx should not be nil")
}
view.holder.addIndexFromField(view.idx)
return view.setValue(tx, columnID, bsig.BitDepth, baseValue)
}
@ -1761,6 +1772,10 @@ func (f *Field) importRoaring(ctx context.Context, tx Tx, data []byte, shard uin
return nil
}
func (f *Field) GetIndex() *Index {
return f.idx
}
func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte, shard uint64, viewName string, block int) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Field.importRoaringOverwrite")
defer span.Finish()
@ -1787,7 +1802,7 @@ func (f *Field) importRoaringOverwrite(ctx context.Context, tx Tx, data []byte,
switch f.Options().Type {
case FieldTypeInt, FieldTypeDecimal:
frag.mu.Lock()
if err := frag.calculateMaxRowID(); err != nil {
if err := frag.calculateMaxRowID(tx); err != nil {
return err
}
maxRowID, _, err := frag.maxRow(tx, nil)
@ -2132,6 +2147,9 @@ func isValidCacheType(v string) bool {
}
}
// TODO(jea): why isn't this bits.Len64(x) using import "math/bits"
// That would be much (80x or more) faster and correct if the high bit is set.
//
// bitDepth returns the number of bits required to store a value.
func bitDepth(v uint64) uint {
for i := uint(0); i < 63; i++ {

63
field_internal_test.go
View file

@ -205,10 +205,17 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField {
if err != nil {
t.Fatal(err)
}
field, err := NewField(NewHolder(DefaultPartitionN), path, "i", "f", opts)
h := NewHolder(DefaultPartitionN)
h.Path = path
idx, err := h.CreateIndex("i", IndexOptions{})
if err != nil {
panic(err)
}
field, err := NewField(h, path, "i", "f", opts)
if err != nil {
t.Fatal(err)
}
field.idx = idx
return &TestField{Field: field}
}
@ -223,6 +230,9 @@ func OpenField(t *testing.T, opts FieldOption) *TestField {
// Close closes the field and removes the underlying data.
func (f *TestField) Close() error {
if f.idx != nil {
panicOn(f.idx.Txf.CloseIndex(f.idx))
}
defer os.RemoveAll(f.Path())
return f.Field.Close()
}
@ -235,10 +245,17 @@ func (f *TestField) Reopen() error {
}
path, index, name := f.Path(), f.Index(), f.Name()
f.Field, err = NewField(NewHolder(DefaultPartitionN), path, index, name, OptFieldTypeDefault())
h := NewHolder(DefaultPartitionN)
h.Path = path
idx, err := h.CreateIndex(index, IndexOptions{})
if err != nil {
return err
}
f.Field, err = NewField(h, path, index, name, OptFieldTypeDefault())
if err != nil {
return err
}
f.Field.idx = idx
if err := f.Open(); err != nil {
return err
@ -311,7 +328,8 @@ func TestField_RowTime(t *testing.T) {
defer f.Close()
// Obtain transaction.
tx := &RoaringTx{Field: f.Field}
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
if err := f.setTimeQuantum(TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
@ -323,6 +341,12 @@ func TestField_RowTime(t *testing.T) {
f.MustSetBit(tx, 1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC))
f.MustSetBit(tx, 1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC))
panicOn(tx.Commit())
// obtain 2nd transaction to read it back.
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
if r, err := f.RowTime(tx, 1, time.Date(2010, time.November, 5, 12, 0, 0, 0, time.UTC), "Y"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(r.Columns(), []uint64{1, 3, 4, 5}) {
@ -358,6 +382,7 @@ func TestField_RowTime(t *testing.T) {
func TestField_PersistAvailableShards(t *testing.T) {
availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
f := OpenField(t, OptFieldTypeDefault())
defer f.Close()
// bm represents remote available shards.
bm := roaring.NewBitmap(1, 2, 3)
@ -379,6 +404,7 @@ func TestField_PersistAvailableShards(t *testing.T) {
func TestField_CorruptAvailableShards(t *testing.T) {
availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
f := OpenField(t, OptFieldTypeDefault())
defer f.Close()
// bm represents remote available shards.
bm := roaring.NewBitmap(1, 2, 3)
@ -411,6 +437,7 @@ func TestField_CorruptAvailableShards(t *testing.T) {
func TestField_TruncatedAvailableShards(t *testing.T) {
availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
f := OpenField(t, OptFieldTypeDefault())
defer f.Close()
// bm represents remote available shards.
bm := roaring.NewBitmap(1, 2, 3)
@ -441,6 +468,7 @@ func TestField_TruncatedAvailableShards(t *testing.T) {
func TestField_PersistAvailableShardsFootprint(t *testing.T) {
availableShardFileFlushDuration.Set(200 * time.Millisecond) //shorten the default time to force a file write
f := OpenField(t, OptFieldTypeDefault())
defer f.Close()
// bm represents remote available shards.
bm := roaring.NewBitmap()
@ -554,6 +582,7 @@ func TestField_ApplyOptions(t *testing.T) {
// to result in a value of 9 instead of 1.
func TestBSIGroup_importValue(t *testing.T) {
f := OpenField(t, OptFieldTypeInt(-100, 200))
defer f.Close()
options := &ImportOptions{}
for i, tt := range []struct {
@ -581,22 +610,31 @@ func TestBSIGroup_importValue(t *testing.T) {
[]uint64{100},
},
} {
tx := &RoaringTx{Field: f.Field}
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
// can't do this, we are in a loop, not a function:
// defer tx.Rollback()
if err := f.importValue(tx, tt.columnIDs, tt.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(tx.Commit())
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
// no, same reason as above: defer tx.Rollback()
if row, err := f.Range(tx, f.name, pql.EQ, tt.checkVal); err != nil {
t.Fatalf("test %d, getting range: %s", i, err.Error())
} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
}
}
tx.Rollback()
} // loop
}
func TestIntField_MinMaxForShard(t *testing.T) {
f := OpenField(t, OptFieldTypeInt(-100, 200))
defer f.Close()
options := &ImportOptions{}
for i, test := range []struct {
@ -648,12 +686,17 @@ func TestIntField_MinMaxForShard(t *testing.T) {
},
} {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
tx := &RoaringTx{Field: f.Field}
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
if err := f.importValue(tx, test.columnIDs, test.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(tx.Commit())
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
maxvc, err := f.MaxForShard(tx, 0, nil)
if err != nil {
t.Fatalf("getting max for shard: %v", err)
@ -753,6 +796,7 @@ func TestDecimalField_MinMaxBoundaries(t *testing.T) {
func TestDecimalField_MinMaxForShard(t *testing.T) {
f := OpenField(t, OptFieldTypeDecimal(3))
defer f.Close()
options := &ImportOptions{}
for i, test := range []struct {
@ -804,12 +848,17 @@ func TestDecimalField_MinMaxForShard(t *testing.T) {
},
} {
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
tx := &RoaringTx{Field: f.Field}
tx := f.idx.Txf.NewTx(Txo{Write: writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
if err := f.importFloatValue(tx, test.columnIDs, test.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(tx.Commit())
tx = f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field})
defer tx.Rollback()
maxvc, err := f.MaxForShard(tx, 0, nil)
if err != nil {
t.Fatalf("getting max for shard: %v", err)

40
field_test.go
View file

@ -25,6 +25,8 @@ import (
"github.com/pilosa/pilosa/v2/test"
)
var panicOn = pilosa.PanicOn
// Ensure a field can set & read a bsiGroup value.
func TestField_SetValue(t *testing.T) {
t.Run("OK", func(t *testing.T) {
@ -35,7 +37,10 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxPilosa := f.Field.GetIndex()
tx := idxPilosa.NewTx(pilosa.Txo{Write: writable, Index: idxPilosa, Field: f.Field})
defer tx.Rollback()
// Set value on field.
if changed, err := f.SetValue(tx, 100, 21); err != nil {
@ -69,7 +74,9 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Set value.
if changed, err := f.SetValue(tx, 100, 21); err != nil {
@ -103,7 +110,9 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Set value.
if _, err := f.SetValue(tx, 100, 21); err != pilosa.ErrBSIGroupNotFound {
@ -119,7 +128,9 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Set value.
if _, err := f.SetValue(tx, 100, 15); err != pilosa.ErrBSIGroupValueTooLow {
@ -135,7 +146,9 @@ func TestField_SetValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Set value.
if _, err := f.SetValue(tx, 100, 31); err != pilosa.ErrBSIGroupValueTooHigh {
@ -204,7 +217,9 @@ func TestField_AvailableShards(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Set values on shards 0 & 2, and verify.
if _, err := f.SetBit(tx, 0, 100, nil); err != nil {
@ -214,6 +229,7 @@ func TestField_AvailableShards(t *testing.T) {
} else if diff := cmp.Diff(f.AvailableShards().Slice(), []uint64{0, 2}); diff != "" {
t.Fatal(diff)
}
panicOn(tx.Commit())
// Set remote shards and verify.
if err := f.AddRemoteAvailableShards(roaring.NewBitmap(1, 2, 4)); err != nil {
@ -244,7 +260,9 @@ func TestField_ClearValue(t *testing.T) {
if err != nil {
t.Fatal(err)
}
tx := &pilosa.RoaringTx{Field: f.Field}
idxP := f.Field.GetIndex()
tx := idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Set value on field.
if changed, err := f.SetValue(tx, 100, 21); err != nil {
@ -252,6 +270,9 @@ func TestField_ClearValue(t *testing.T) {
} else if !changed {
t.Fatal("expected change")
}
panicOn(tx.Commit())
tx = idxP.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idxP, Field: f.Field})
// Read value.
if value, exists, err := f.Value(tx, 100); err != nil {
@ -261,12 +282,17 @@ func TestField_ClearValue(t *testing.T) {
} else if !exists {
t.Fatal("expected value to exist")
}
tx.Rollback()
tx = idxP.Txf.NewTx(pilosa.Txo{Write: writable, Index: idxP, Field: f.Field})
if changed, err := f.ClearValue(tx, 100); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal(err)
}
panicOn(tx.Commit())
tx = idxP.Txf.NewTx(pilosa.Txo{Write: !writable, Index: idxP, Field: f.Field})
defer tx.Rollback()
// Read value.
if _, exists, err := f.Value(tx, 100); err != nil {

173
fragment.go
View file

@ -108,6 +108,9 @@ type fragment struct {
view string
shard uint64
// idx cached to avoid repeatedly looking it up everywhere.
idx *Index
// parent holder, used to find snapshot queue, etc.
holder *Holder
@ -180,6 +183,10 @@ type fragment struct {
// newFragment returns a new instance of Fragment.
func newFragment(holder *Holder, path, index, field, view string, shard uint64, flags byte) *fragment {
idx := holder.Index(index)
if idx == nil {
panic(fmt.Sprintf("holder=%#v but got nil idx back from holder!", holder))
}
f := &fragment{
path: path,
index: index,
@ -187,6 +194,7 @@ func newFragment(holder *Holder, path, index, field, view string, shard uint64,
view: view,
shard: shard,
flags: flags,
idx: idx,
CacheType: DefaultCacheType,
CacheSize: DefaultCacheSize,
@ -250,7 +258,9 @@ func (f *fragment) Open() error {
f.checksums = make(map[int][]byte)
// Read last bit to determine max row.
return f.calculateMaxRowID()
tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f})
defer tx.Rollback()
return f.calculateMaxRowID(tx)
}(); err != nil {
f.close()
return err
@ -398,6 +408,15 @@ func (f *fragment) inspectStorage(data []byte, file *os.File, newGen generation,
// logic is now mostly in importStorage (reading in a bitmap) and applyStorage
// (remapping an existing bitmap to match a new backing store).
func (f *fragment) openStorage(unmarshalData bool) error {
if !f.idx.NeedsSnapshot() {
f.gen = &NopGeneration{}
f.rowCache = &simpleCache{make(map[uint64]*Row)}
f.currdata = struct{ from, to uintptr }{}
f.prevdata = f.currdata
return nil // openStorage becomes a noop under RBF, Badger, etc.
}
// Create a roaring bitmap to serve as storage for the shard.
if f.storage == nil {
f.storage = roaring.NewFileBitmap()
@ -473,10 +492,16 @@ func (f *fragment) openCache() error {
return nil
}
tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx, Fragment: f})
defer tx.Rollback()
// Read in all rows by ID.
// This will cause them to be added to the cache.
for _, id := range pb.IDs {
n := f.storage.CountRange(id*ShardWidth, (id+1)*ShardWidth)
n, err := tx.CountRange(f.index, f.field, f.view, f.shard, id*ShardWidth, (id+1)*ShardWidth)
if err != nil {
return errors.Wrap(err, "CountRange")
}
f.cache.BulkAdd(id, n)
}
f.cache.Invalidate()
@ -583,7 +608,7 @@ func (f *fragment) rowFromStorage(tx Tx, rowID uint64) (*Row, error) {
row := &Row{
segments: []rowSegment{{
data: data, // this data contains BadgerTx data, which should not survive Txn commit.
data: data,
shard: f.shard,
writable: true,
}},
@ -598,7 +623,11 @@ func (f *fragment) rowFromStorage(tx Tx, rowID uint64) (*Row, error) {
func (f *fragment) setBit(tx Tx, rowID, columnID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
// handle mutux field type
if f.mutexVector != nil {
if err := f.handleMutex(tx, rowID, columnID); err != nil {
@ -680,7 +709,11 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo
func (f *fragment) clearBit(tx Tx, rowID, columnID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
changed, err = f.unprotectedClearBit(tx, rowID, columnID)
return err
})
@ -739,7 +772,11 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b
func (f *fragment) setRow(tx Tx, row *Row, rowID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
changed, err = f.unprotectedSetRow(tx, row, rowID)
return err
})
@ -802,7 +839,11 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo
func (f *fragment) clearRow(tx Tx, rowID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
changed, err = f.unprotectedClearRow(tx, rowID)
return err
})
@ -845,7 +886,11 @@ func (f *fragment) unprotectedClearRow(tx Tx, rowID uint64) (changed bool, err e
// This updates both the on-disk storage and the in-cache bitmap.
func (f *fragment) unprotectedClearBlock(tx Tx, block int) (changed bool, err error) {
firstRow := uint64(block * HashBlockSize)
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
var rowChanged bool
for rowID := uint64(firstRow); rowID < firstRow+HashBlockSize; rowID++ {
if changed, err := f.unprotectedClearRow(tx, rowID); err != nil {
@ -953,8 +998,11 @@ func (f *fragment) positionsForValue(columnID uint64, bitDepth uint, value int64
func (f *fragment) setValueBase(tx Tx, columnID uint64, bitDepth uint, value int64, clear bool) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
err = f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err = f.gen.Transaction(wp, func() error {
// Convert value to an unsigned representation.
uvalue := uint64(value)
if value < 0 {
@ -1291,8 +1339,12 @@ func (f *fragment) maxRow(tx Tx, filter *Row) (uint64, uint64, error) {
// calculateMaxRowID determines the field's maxRowID value based
// on the contents of its storage, and sets the struct argument.
func (f *fragment) calculateMaxRowID() (err error) {
f.maxRowID = f.storage.Max() / ShardWidth
func (f *fragment) calculateMaxRowID(tx Tx) (err error) {
max, err := tx.Max(f.index, f.field, f.view, f.shard)
if err != nil {
return err
}
f.maxRowID = max / ShardWidth
return nil
}
@ -1484,7 +1536,6 @@ func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality
if err != nil {
return nil, err
}
// Create predicate without sign bit.
upredicate := absInt64(predicate)
@ -1492,7 +1543,6 @@ func (f *fragment) rangeGT(tx Tx, bitDepth uint, predicate int64, allowEquality
if err != nil {
return nil, err
}
switch {
case predicate == 0 && !allowEquality:
// Match all positive numbers except zero.
@ -2217,7 +2267,11 @@ func (f *fragment) bulkImportStandard(tx Tx, rowIDs, columnIDs []uint64, options
// operations to the op log.
func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64]struct{}) error {
//tx.AddN()
err := f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err := f.gen.Transaction(wp, func() error { // segfault
if len(set) > 0 {
f.stats.Count(MetricImportingN, int64(len(set)), 1)
@ -2453,14 +2507,18 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b
span, ctx := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits")
var changed int
var rowSet map[uint64]int
err := f.gen.Transaction(&f.storage.OpWriter, func() (err error) {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err := f.gen.Transaction(wp, func() (err error) {
var rit roaring.RoaringIterator
rit, err = roaring.NewRoaringIterator(data)
if err != nil {
return err
}
changed, rowSet, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, rit, clear, true, rowSize)
changed, rowSet, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, rit, clear, true, rowSize, nil)
return err
})
@ -2550,6 +2608,9 @@ func track(start time.Time, message string, stats stats.StatsClient, logger logg
// snapshot does the actual snapshot operation. it does not check or care
// about f.snapshotPending.
func (f *fragment) snapshot() (err error) {
if !f.idx.NeedsSnapshot() {
return nil
}
if !f.open {
return errors.New("snapshot request on closed fragment")
}
@ -2688,31 +2749,16 @@ func (f *fragment) WriteTo(w io.Writer) (n int64, err error) {
return 0, nil
}
// used in shipping the slices across the network for a resize.
func (f *fragment) writeStorageToArchive(tw *tar.Writer) error {
// Open separate file descriptor to read from.
file, err := os.Open(f.path)
tx := f.idx.Txf.NewTx(Txo{Write: !writable, Index: f.idx})
defer tx.Rollback()
file, sz, err := tx.RoaringBitmapReader(f.index, f.field, f.view, f.shard, f.path)
if err != nil {
return errors.Wrap(err, "opening file")
}
defer file.Close()
// Retrieve the current file size under lock so we don't read
// while an operation is appending to the end.
var sz int64
if err := func() error {
f.mu.Lock()
defer f.mu.Unlock()
fi, err := file.Stat()
if err != nil {
return errors.Wrap(err, "statting")
}
sz = fi.Size()
return nil
}(); err != nil {
return err
}
defer file.Close()
// Write archive header.
if err := tw.WriteHeader(&tar.Header{
@ -2779,9 +2825,15 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) {
// Process file based on file name.
switch hdr.Name {
case "data":
if err := f.readStorageFromArchive(tr); err != nil {
idx := f.holder.Index(f.index)
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
defer tx.Rollback()
if err := f.fillFragmentFromArchive(tx, tr); err != nil {
return 0, errors.Wrap(err, "reading storage")
}
if err := tx.Commit(); err != nil {
return 0, errors.Wrap(err, "Commit after tx.ReadFragmentFromArchive")
}
case "cache":
if err := f.readCacheFromArchive(tr); err != nil {
return 0, errors.Wrap(err, "reading cache")
@ -2794,7 +2846,40 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) {
return 0, nil
}
// should be morally equivalent to fragment.readStorageFromArchive()
// below for RoaringTx, but also work on any Tx because it uses
// tx.ImportRoaringBits().
func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error {
// this is reading from inside a tarball, so definitely no need
// to close it here.
data, err := ioutil.ReadAll(r)
if err != nil {
return errors.Wrap(err, "fillFragmentFromArchive ioutil.ReadAll(r)")
}
if len(data) == 0 {
return nil
}
// For reference, compare to what fragment.go:313 fragment.importStorage() does.
clear := false
log := false
rowSize := uint64(0)
itr, err := roaring.NewRoaringIterator(data)
if err != nil {
return errors.Wrap(err, "fillFragmentFromArchive NewRoaringIterator")
}
changed, rowSet, err := tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, itr, clear, log, rowSize, data)
_, _ = changed, rowSet
if err != nil {
return errors.Wrap(err, "fillFragmentFromArchive ImportRoaringBits")
}
return nil
}
func (f *fragment) readStorageFromArchive(r io.Reader) error {
// Create a temporary file to copy into.
path := f.path + copyExt
file, err := os.Create(path)
@ -2808,6 +2893,12 @@ func (f *fragment) readStorageFromArchive(r io.Reader) error {
return errors.Wrap(err, "copying")
}
// TODO(jea): isn't this next Rename a file handle leak?
// try closing first
if err := f.closeStorage(); err != nil {
return errors.Wrap(err, "closeStorage-prior-to-Rename-and-openStorage")
}
// Move snapshot to data file location.
if err := os.Rename(path, f.path); err != nil {
return errors.Wrap(err, "renaming")
@ -2934,13 +3025,15 @@ func (f *fragment) rows(ctx context.Context, tx Tx, start uint64, filters ...row
// unprotectedRows calls rows without grabbing the mutex.
func (f *fragment) unprotectedRows(ctx context.Context, tx Tx, start uint64, filters ...rowFilter) ([]uint64, error) {
rows := make([]uint64, 0)
startKey := rowToKey(start)
i, _, err := tx.ContainerIterator(f.index, f.field, f.view, f.shard, startKey)
if err != nil {
return nil, err
} else if i == nil {
return rows, nil
}
defer i.Close() // must close iterators allocated on a Tx
rows := make([]uint64, 0)
var lastRow uint64 = math.MaxUint64
// Loop over the existing containers.

1116
fragment_internal_test.go
View file

File diff suppressed because it is too large Load diff

23
generation.go
View file

@ -261,6 +261,7 @@ func (m *mmapGeneration) openFile() (shouldClose bool, err error) {
if err != nil {
return false, err
}
// do we actually want this in every openFile? I don't know.
if err := syscall.Flock(int(m.file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
_ = syswrap.CloseFile(m.file)
@ -438,3 +439,25 @@ func newGeneration(existing generation, path string, readData bool, setup func([
// does get cleaned up.
return &m, nil
}
// NopGeneration is used in fragment.openStorage() to short-circuit
// generation stuff that only applies to RoaringTx; doesn't apply to RBFTx/BadgerTx/etc.
type NopGeneration struct {
}
func (g *NopGeneration) Transaction(w *io.Writer, f func() error) error {
return f()
}
func (g *NopGeneration) Done() {}
func (g *NopGeneration) Generation() int64 {
return 0
}
func (g *NopGeneration) ID() string {
return "NOP"
}
func (g *NopGeneration) Dead() bool {
return true
}
func (g *NopGeneration) Bytes() (ret []byte) {
return
}

7
generation_test.go
View file

@ -53,8 +53,11 @@ func TestGenerationPanic(t *testing.T) {
if unsafe.Pointer(&prevData[0]) == unsafe.Pointer(&newData[0]) {
t.Fatalf("test can't run usefully, didn't get new data pointer")
}
err := f.gen.Transaction(&f.storage.OpWriter, func() error {
var wp *io.Writer
if f.storage != nil {
wp = &f.storage.OpWriter
}
err := f.gen.Transaction(wp, func() error {
prevData[0] = 0x3c
return nil
})

7
go.mod
View file

@ -9,11 +9,8 @@ require (
github.com/benbjohnson/immutable v0.2.0
github.com/boltdb/bolt v1.3.1
github.com/cespare/xxhash v1.1.0
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
github.com/davecgh/go-spew v1.1.1
github.com/dchest/blake2b v1.0.0 // indirect
github.com/dgraph-io/badger v1.6.1-0.20191025180844-32a2548a9d85 // indirect
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/gogo/protobuf v1.2.0
@ -22,8 +19,7 @@ require (
github.com/gorilla/handlers v1.3.0
github.com/gorilla/mux v1.7.0
github.com/hashicorp/memberlist v0.1.3
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2
github.com/molecula/ext v0.0.0-20200103203257-8a458a73e8c2 // indirect
github.com/molecula/extensions v0.0.0-20191218165536-562244600fd4
github.com/opentracing/opentracing-go v1.1.0
github.com/pelletier/go-toml v1.2.0
@ -40,7 +36,6 @@ require (
github.com/uber-go/atomic v1.4.0 // indirect
github.com/uber/jaeger-client-go v2.16.0+incompatible
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
github.com/willoch/tago v0.0.0-20180311150625-8f2f8e8900dc // indirect
github.com/zeebo/blake3 v0.0.4
go.uber.org/atomic v1.4.0 // indirect
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect

36
go.sum
View file

@ -26,9 +26,6 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
@ -40,18 +37,8 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dchest/blake2b v1.0.0 h1:KK9LimVmE0MjRl9095XJmKqZ+iLxWATvlcpVFRtaw6s=
github.com/dchest/blake2b v1.0.0/go.mod h1:U034kXgbJpCle2wSk5ybGIVhOSHCVLMDqOzcPEA0F7s=
github.com/dgraph-io/badger v1.6.1-0.20191025180844-32a2548a9d85 h1:oEqDRoxpep5ZlTxrAFc2yg+f0uBdUtkZE0uWsOru5bc=
github.com/dgraph-io/badger v1.6.1-0.20191025180844-32a2548a9d85/go.mod h1:cEjdIw+iaGXuQdsDymXPRcpp8yHXZ6PmwmDJajnVyJc=
github.com/dgraph-io/badger v1.6.1 h1:w9pSFNSdq/JPM1N12Fz/F/bzo993Is1W+Q7HjPzi7yg=
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 h1:JBNM90aGLCiF9iJYvpvayMpYeW498v5ZDZqE2chqZ2A=
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE=
github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl6XJI=
github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM=
github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs=
github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU=
github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA=
github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
@ -115,8 +102,10 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@ -145,7 +134,6 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021 h1:ERLyN4p3KS5Fk2ADsDENm2cq0+Lx6sF1sG8uwRlySpU=
github.com/pilosa/memberlist v0.1.4-0.20190415211605-f6512523c021/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/pilosa/pilosa v1.4.0 h1:nqHNIK4nDslFnem3yDp9R+6TgLdlkY9WdJD88Z83T8U=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -180,29 +168,25 @@ github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38=
github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
@ -211,15 +195,13 @@ github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW
github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/willoch/tago v0.0.0-20180311150625-8f2f8e8900dc h1:Jsemerl8qK30jGNdYlxGZpZk9RjB4pqvezJgxqUgy30=
github.com/willoch/tago v0.0.0-20180311150625-8f2f8e8900dc/go.mod h1:9WHA/f8A/TRK+WQQZhqx47In4pnIhMTH6UrsgqqgsVQ=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0=
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/blake3 v0.0.4-0.20200428182842-252974700486 h1:yh0zEy8it58x/IPNtKKuvKUkxSIaq4s5XiRSd40JuYs=
github.com/zeebo/blake3 v0.0.4-0.20200428182842-252974700486/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34=
github.com/zeebo/blake3 v0.0.4 h1:vtZ4X8B2lKXZFg2Xyg6Wo36mvmnJvc2VQYTtA4RDCkI=
github.com/zeebo/blake3 v0.0.4/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34=
github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05 h1:4pW5fMvVkrgkMXdvIsVRRTs69DWYA8uNNQsu1stfVKU=
github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05/go.mod h1:Gr+78ptB0MwXxm//LBaEvBiaXY7hXJ6KGe2V32X2F6E=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@ -243,8 +225,6 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU=
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -263,8 +243,6 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8=
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
@ -292,6 +270,7 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
@ -302,4 +281,3 @@ modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I=
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE=
modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130=

48
holder.go
View file

@ -528,9 +528,9 @@ func (h *Holder) Open() error {
if h.isCoordinator() {
index.createdAt = timestamp()
err = index.OpenWithTimestamp()
err = index.OpenWithTimestamp(false)
} else {
err = index.Open()
err = index.Open(false)
}
if err != nil {
if err == ErrName {
@ -626,6 +626,17 @@ func (h *Holder) BeginTx(writable bool, index *Index) (Tx, error) {
return index.Txf.NewTx(Txo{Write: writable, Index: index}), nil
}
func (h *Holder) NeedsSnapshot() bool {
h.mu.RLock()
defer h.mu.RUnlock()
for _, idx := range h.indexes {
if idx.NeedsSnapshot() {
return true
}
}
return false
}
// HasData returns true if Holder contains at least one index.
// This is used to determine if the rebalancing of data is necessary
// when a node joins the cluster.
@ -802,7 +813,20 @@ func (h *Holder) applyCreatedAt(indexes []*IndexInfo) {
}
// IndexPath returns the path where a given index is stored.
func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
func (h *Holder) IndexPath(name string) string {
return filepath.Join(h.Path, name)
}
// HolderPathFromIndexPath is
// used by test/index.go:71 in test.Index.Reopen() to get the right
// path into a test Holder that doesn't know its own proper path.
// If the Holder changes index paths to being something other than
// holderPath + "/" + indexName, this will need adjusting too.
func (h *Holder) HolderPathFromIndexPath(indexPath, indexName string) string {
n := len(indexPath)
hpath2 := indexPath[:n-(len(indexName)+1)]
return hpath2
}
// Index returns the index by name.
func (h *Holder) Index(name string) *Index {
@ -811,7 +835,9 @@ func (h *Holder) Index(name string) *Index {
return h.index(name)
}
func (h *Holder) index(name string) *Index { return h.indexes[name] }
func (h *Holder) index(name string) *Index {
return h.indexes[name]
}
// Indexes returns a list of all indexes in the holder.
func (h *Holder) Indexes() []*Index {
@ -867,7 +893,7 @@ func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
index.keys = opt.Keys
index.trackExistence = opt.TrackExistence
if err = index.Open(); err != nil {
if err = index.Open(true); err != nil {
return nil, errors.Wrap(err, "opening")
}
if err = index.saveMeta(); err != nil {
@ -1799,3 +1825,15 @@ func (h *Holder) Process(ctx context.Context, op HolderOperator) (err error) {
}
return nil
}
// used by Index.openFields(), enabling Tx / Txf by telling
// the holder about its own indexes.
func (h *Holder) addIndexFromField(idx *Index) {
h.mu.Lock()
h.indexes[idx.Name()] = idx
h.mu.Unlock()
}
func (h *Holder) unprotectedAddIndexFromField(idx *Index) {
h.indexes[idx.Name()] = idx
}

21
holder_test.go
View file

@ -32,6 +32,7 @@ import (
)
func TestHolder_Open(t *testing.T) {
t.Run("ErrIndexName", func(t *testing.T) {
h := test.MustOpenHolder()
@ -165,6 +166,8 @@ func TestHolder_Open(t *testing.T) {
})
t.Run("ErrFragmentStoragePermission", func(t *testing.T) {
roaringOnlyTest(t)
if os.Geteuid() == 0 {
t.Skip("Skipping permissions test since user is root.")
}
@ -201,6 +204,8 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) {
roaringOnlyTest(t)
h := test.MustOpenHolder()
defer h.Close()
@ -233,6 +238,8 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFragmentStorageRecoverable", func(t *testing.T) {
roaringOnlyTest(t)
h := test.MustOpenHolder()
defer h.Close()
@ -397,6 +404,8 @@ func TestHolder_HasData(t *testing.T) {
// Ensure holder can delete an index and its underlying files.
func TestHolder_DeleteIndex(t *testing.T) {
skipForRBF(t)
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -572,7 +581,6 @@ func TestHolderSyncer_BlockIteratorLimits(t *testing.T) {
// Leave the third replica empty to force a block merge.
//
err = c[0].Server.SyncData()
if err != nil {
t.Fatalf("syncing node 0: %v", err)
@ -692,6 +700,8 @@ func TestHolderSyncer_TimeQuantum(t *testing.T) {
// Ensure holder can sync integer views with a remote holder.
func TestHolderSyncer_IntField(t *testing.T) {
skipForRBF(t)
t.Run("BasicSync", func(t *testing.T) {
c := test.MustNewCluster(t, 2)
c[0].Config.Cluster.ReplicaN = 2
@ -723,24 +733,17 @@ func TestHolderSyncer_IntField(t *testing.T) {
hldr0.SetValue("i", "f", 1, 1)
// in c0 expect the 1 bit
//idx0.Dump("in c0, before SyncData")
// Set data on node1. columnID=2, value=2
idx1 := hldr1.SetValue("i", "f", 2, 2)
_ = idx1
//idx1.Dump("in c1, before SyncData")
//vv("before c[0] SyncData")
err = c[0].Server.SyncData()
if err != nil {
t.Fatalf("syncing node 0: %v", err)
}
//vv("after c[0] SyncData")
// expect 3 rows, the 1 bit + 2 rows for the 2 value as BSI. But, we only see that c0 overwrote c1.
//idx0.Dump("in c0, after syncData")
//idx1.Dump("in c1, after syncData")
// Problem is: data at c1 was replaced by c0, instead of being merged with existing c1.
// Problem is: data at c0 did not receive and merge the c1 data.
@ -810,8 +813,6 @@ func TestHolderSyncer_IntField(t *testing.T) {
}
// dump the badger keys for both c0 and c1
//vv("in c0, allkeys = '%v'", idx0.StringifiedBadgerKeys(nil))
//vv("in c1, allkeys = '%v'", c[1].index.StringifiedBadgerKeys())
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {

93
index.go
View file

@ -72,8 +72,21 @@ type Index struct {
Txf *TxFactory
}
// NewIndex returns a new instance of Index.
// OpenIndex opens or starts a new Index on path. Path
// can be empty.
func OpenIndex(holder *Holder, path, name string) (*Index, error) {
openExisting := true
return openOrCreateNewIndex(holder, path, name, openExisting)
}
// NewIndex returns a new instance of Index at path. It will erase anything
// old already in path.
func NewIndex(holder *Holder, path, name string) (*Index, error) {
openExisting := false
return openOrCreateNewIndex(holder, path, name, openExisting)
}
func openOrCreateNewIndex(holder *Holder, path, name string, openExisting bool) (*Index, error) {
// Emulate what the spf13/cobra does, letting env vars override
// the defaults, because we may be under a simple "go test" run where
@ -99,16 +112,16 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
}
}
txf, err := newTxFactory(txsrc, path)
if err != nil {
return nil, errors.Wrap(err, "creating newTxFactory")
}
err = validateName(name)
err := validateName(name)
if err != nil {
return nil, errors.Wrap(err, "validating name")
}
txf, err := NewTxFactory(txsrc, holder.Path, name, openExisting)
if err != nil {
return nil, errors.Wrap(err, "creating newTxFactory")
}
idx := &Index{
path: path,
name: name,
@ -134,6 +147,14 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
return idx, nil
}
func (i *Index) NewTx(txo Txo) Tx {
return i.Txf.NewTx(txo)
}
func (i *Index) NeedsSnapshot() bool {
return i.Txf.NeedsSnapshot()
}
// CreatedAt is an timestamp for a specific version of an index.
func (i *Index) CreatedAt() int64 {
i.mu.RLock()
@ -148,7 +169,9 @@ func (i *Index) Name() string { return i.name }
func (i *Index) QualifiedName() string { return i.qualifiedName }
// Path returns the path the index was initialized with.
func (i *Index) Path() string { return i.path }
func (i *Index) Path() string {
return i.path
}
// TranslateStorePath returns the translation database path for a partition.
func (i *Index) TranslateStorePath(partitionID int) string {
@ -181,12 +204,12 @@ func (i *Index) options() IndexOptions {
}
// Open opens and initializes the index.
func (i *Index) Open() error { return i.open(false) }
func (i *Index) Open(haveHolderLock bool) error { return i.open(false, haveHolderLock) }
// OpenWithTimestamp opens and initializes the index and set a new CreatedAt timestamp for fields.
func (i *Index) OpenWithTimestamp() error { return i.open(true) }
func (i *Index) OpenWithTimestamp(haveHolderLock bool) error { return i.open(true, haveHolderLock) }
func (i *Index) open(withTimestamp bool) (err error) {
func (i *Index) open(withTimestamp, haveHolderLock bool) (err error) {
// Ensure the path exists.
i.holder.Logger.Debugf("ensure index path exists: %s", i.path)
if err := os.MkdirAll(i.path, 0777); err != nil {
@ -200,7 +223,7 @@ func (i *Index) open(withTimestamp bool) (err error) {
}
i.holder.Logger.Debugf("open fields for index: %s", i.name)
if err := i.openFields(withTimestamp); err != nil {
if err := i.openFields(withTimestamp, haveHolderLock); err != nil {
return errors.Wrap(err, "opening fields")
}
@ -243,7 +266,7 @@ func (i *Index) open(withTimestamp bool) (err error) {
var indexQueue = make(chan struct{}, 8)
// openFields opens and initializes the fields inside the index.
func (i *Index) openFields(withTimestamp bool) error {
func (i *Index) openFields(withTimestamp, haveHolderLock bool) error {
f, err := os.Open(i.path)
if err != nil {
return errors.Wrap(err, "opening directory")
@ -273,7 +296,31 @@ fileLoop:
<-indexQueue
}()
i.holder.Logger.Debugf("open field: %s", fi.Name())
mu.Lock()
// i.holder needs to know about its index i for the Txf to work.
//
// We face either a deadlock or a race here.
//
// We get a deadlock in TestIndex_CreateField/"BSIFields"/"OK"
// if we call addIndexFromField, because in that test
// we get here while already holding i.holder.mu.
//
// On the other had, we get races on other tests
// such as TestExecutor_Execute_Existence/Row
// if we call unprotectedAddIndexFromField which does
// not lock i.holder.mu.
//
// The resolution was to have the goroutines that are holding
// the lock already tell us. That is the haveHolderLock
// argument.
if haveHolderLock {
i.holder.unprotectedAddIndexFromField(i)
} else {
i.holder.addIndexFromField(i)
}
fld, err := i.newField(i.fieldPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if withTimestamp {
fld.createdAt = timestamp()
@ -287,6 +334,7 @@ fileLoop:
// up a foreign index.
fld.holder = i.holder
// open all the views
if err := fld.Open(); err != nil {
return fmt.Errorf("open field: name=%s, err=%s", fld.Name(), err)
}
@ -546,6 +594,9 @@ func (i *Index) createField(name string, opt *FieldOptions) (*Field, error) {
// Add to index's field lookup.
i.fields[name] = f
// enable Txf to find the index in field_test.go TestField_SetValue
f.idx = i
// Kick off the field's translation sync process.
if err := i.translationSyncer.Reset(); err != nil {
return nil, errors.Wrap(err, "resetting translation syncer")
@ -559,6 +610,7 @@ func (i *Index) newField(path, name string) (*Field, error) {
if err != nil {
return nil, err
}
f.idx = i
f.Stats = i.Stats
f.broadcaster = i.broadcaster
f.rowAttrStore = i.newAttrStore(filepath.Join(f.path, ".data"))
@ -582,9 +634,8 @@ func (i *Index) DeleteField(name string) error {
return errors.Wrap(err, "closing")
}
// Delete field directory.
if err := os.RemoveAll(i.fieldPath(name)); err != nil {
return errors.Wrap(err, "removing directory")
if err := i.Txf.DeleteFieldFromStore(i.name, name, i.fieldPath(name)); err != nil {
return errors.Wrap(err, "Txf.DeleteFieldFromStore")
}
// If the field being deleted is the existence field,
@ -661,3 +712,13 @@ type importValueData struct {
func FormatQualifiedIndexName(index string) string {
return fmt.Sprintf("%s\x00", index)
}
// Dump prints to stdout the contents of the roaring Containers
// stored in idx. Mostly for debugging.
func (idx *Index) Dump(label string) {
fileline := FileLine(2)
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx})
defer tx.Rollback()
fmt.Printf("\n%v Index.Dump('%v') for index '%v':\n", fileline, label, idx.name)
tx.Dump()
}

9
index_internal_test.go
View file

@ -25,7 +25,10 @@ func mustOpenIndex(opt IndexOptions) *Index {
if err != nil {
panic(err)
}
index, err := NewIndex(NewHolder(1), path, "i")
h := NewHolder(1)
h.Path = path
index, err := h.CreateIndex("i", opt)
if err != nil {
panic(err)
}
@ -33,7 +36,7 @@ func mustOpenIndex(opt IndexOptions) *Index {
index.keys = opt.Keys
index.trackExistence = opt.TrackExistence
if err := index.Open(); err != nil {
if err := index.Open(false); err != nil {
panic(err)
}
return index
@ -44,7 +47,7 @@ func (i *Index) reopen() error {
if err := i.Close(); err != nil {
return err
}
if err := i.Open(); err != nil {
if err := i.Open(false); err != nil {
return err
}
return nil

2
license.exceptions
View file

@ -10,3 +10,5 @@
./logger/filewriter.go
./logger/filewriter_test.go
./vprint.go
./rbf/vprint.go
./cmd/loader/vprint.go

1
like_test.go
View file

@ -20,7 +20,6 @@ import (
)
func TestPlanLike(t *testing.T) {
t.Parallel()
cases := []struct {
name string

6
mmap_test.go
View file

@ -32,11 +32,11 @@ type cv struct {
func forceSnapshotsCheckMapping(t *testing.T) {
depth := uint(6)
f, idx := mustOpenBSIFragment("i", "f", viewStandard, 0)
_ = idx
f.Logger = logger.NewLogfLogger(t)
defer f.Clean(t)
tx := &RoaringTx{fragment: f}
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f})
defer tx.Rollback()
for i := 0; i < f.MaxOpN; i++ {
_, _ = f.setBit(tx, 0, uint64(32*i))
@ -86,6 +86,8 @@ func forceSnapshotsCheckMapping(t *testing.T) {
// in newGeneration in generation.go. So this is probably useless but it's
// a failure mode we've been bitten by once...
func TestMmapBehavior(t *testing.T) {
skipForRBF(t)
var changed bool
var original uint64
defer func() {

7
pilosa_test.go
View file

@ -15,6 +15,7 @@
package pilosa_test
import (
"os"
"strings"
"testing"
@ -55,3 +56,9 @@ func TestAddressWithDefaults(t *testing.T) {
}
}
}
func skipForRBF(tb testing.TB) {
if os.Getenv("PILOSA_TXSRC") == "rbf" {
tb.Skip("skip for RBF")
}
}

34
pql/ast.go
View file

@ -22,8 +22,6 @@ import (
"strconv"
"strings"
"time"
"github.com/molecula/ext"
)
// Query represents a PQL query.
@ -345,7 +343,8 @@ var callInfoByFunc = map[string]callInfo{
"Row": {allowUnknown: true},
"Range": {allowUnknown: true},
"Distinct": {allowUnknown: true},
"Distinct": {allowUnknown: true, callType: PrecallGlobal},
"Condition": {allowUnknown: true},
// allow only "field=X" cases with string field names
"Max": allowField,
@ -462,30 +461,6 @@ var callInfoByFunc = map[string]callInfo{
},
}
// RegisterPluginFuncs adds arg validation for plugin funcs. Not very good
// arg validation.
func RegisterPluginFuncs(ops []ext.BitmapOp) {
for _, op := range ops {
// ignore overlap for now. This should change.
if _, ok := callInfoByFunc[op.Name]; ok {
continue
}
ci := callInfo{allowUnknown: true}
if len(op.Reserved) > 0 {
// mark these as valid/known reserved words
ci.prototypes = make(map[string]interface{})
for _, res := range op.Reserved {
ci.prototypes[res] = nil
}
}
t := op.Func.BitmapOpType()
if t.Precall == ext.OpPrecallGlobal {
ci.callType = PrecallGlobal
}
callInfoByFunc[op.Name] = ci
}
}
// CheckCallInfo tries to validate that arguments are correct and valid for the
// given call. It does not guarantee checking all possible errors; for instance,
// if an argument is a field name, CheckCallInfo can't validate that the field
@ -505,6 +480,11 @@ func (c *Call) CheckCallInfo() error {
if !ok && strings.HasPrefix(k, "_") {
return fmt.Errorf("'%s': unknown reserved arg '%s'", c.String(), k)
}
if call, ok := v.(*Call); ok {
if err := call.CheckCallInfo(); err != nil {
return err
}
}
if acceptable == nil {
continue
}

97
rbf/blake3.go Normal file
View file

@ -0,0 +1,97 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rbf
import (
"encoding/binary"
"fmt"
"sync"
cryptorand "crypto/rand"
"github.com/zeebo/blake3"
)
// Blake3Hasher is a thread/goroutine safe way to
// obtain a blake3 cryptographic hash of input []byte.
// Reference https://github.com/BLAKE3-team/BLAKE3
// suggests it is 6x faster than BLAKE2B.
// The Go github.com/zeebo/blake3 version is
// AVX2 and SSE4.1 accelerated.
type Blake3Hasher struct {
hasher *blake3.Hasher
hasherMu sync.Mutex
}
// NewBlake3Hasher returns a new Blake3Hasher.
func NewBlake3Hasher() *Blake3Hasher {
return &Blake3Hasher{
hasher: blake3.New(),
}
}
// CryptoHash writes the blake3 cryptographic hash of
// input into buffer and returns it.
// Like the standard libary's hash.Hash interface's Sum() method,
// the buffer is re-used and overwritten
// to avoid allocation. The caller determines the byte length of
// the outputCryptohash by the size of the supplied buffer
// slice, and this will be exactly equal to the supplies bytes.
// In this way, shorter or longer hashes can be provided as
// needed.
func (w *Blake3Hasher) CryptoHash(input []byte, buffer []byte) (outputCryptohash []byte) {
w.hasherMu.Lock()
w.hasher.Reset()
// "Write implements part of the hash.Hash interface. It never returns an error."
// -- https://godoc.org/github.com/zeebo/blake3#Hasher.Write
_, _ = w.hasher.Write(input)
// Digest.Read reads data from the hasher into buffer.
// "It always fills the entire buffer and never errors."
// -- https://godoc.org/github.com/zeebo/blake3#Digest
_, _ = w.hasher.Digest().Read(buffer)
// no chance of panic, so avoid any defer cost.
w.hasherMu.Unlock()
return buffer
}
// blake3sum16 might be slower because we allocate a new hasher every time, but
// it is more conenient for writing debug code. It returns
// a 16 byte hash as a hexidecimal string.
func blake3sum16(input []byte) string {
hasher := blake3.New()
_, _ = hasher.Write(input)
var buf [16]byte
_, _ = hasher.Digest().Read(buf[0:])
return fmt.Sprintf("%x", buf)
}
// cryptoRandInt64 uses crypto/rand to get an random int64
func cryptoRandInt64() int64 {
c := 8
b := make([]byte, c)
_, err := cryptorand.Read(b)
if err != nil {
panic(err)
}
r := int64(binary.LittleEndian.Uint64(b))
return r
}
var _ = cryptoRandInt64 // happy linter

27
rbf/cursor.go
View file

@ -84,7 +84,7 @@ func runAdd(runs []roaring.Interval16, v uint16) ([]roaring.Interval16, bool) {
}
return runs, true
}
func checkRun(runs []roaring.Interval16, key uint64) leafCell {
func checkRun(runs []roaring.Interval16, bitN int, key uint64) leafCell {
if len(runs) >= RLEMaxSize {
//convertToBitmap
bitmap := make([]uint64, BitmapN)
@ -123,9 +123,9 @@ func checkRun(runs []roaring.Interval16, key uint64) leafCell {
n += popcount(v)
}
return leafCell{Key: key, N: int(n), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)}
return leafCell{Key: key, N: int(n), BitN: int(n), Type: ContainerTypeBitmap, Data: fromArray64(bitmap)}
}
return leafCell{Key: key, N: len(runs), Type: ContainerTypeRLE, Data: fromInterval16(runs)}
return leafCell{Key: key, N: len(runs), BitN: int(bitN + 1), Type: ContainerTypeRLE, Data: fromInterval16(runs)}
}
// Add sets a bit on the underlying bitmap.
@ -136,7 +136,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
if exact, err := c.Seek(hi); err != nil {
return false, err
} else if !exact {
return true, c.putLeafCell(leafCell{Key: hi, Type: ContainerTypeArray, N: 1, Data: fromArray16([]uint16{lo})})
return true, c.putLeafCell(leafCell{Key: hi, Type: ContainerTypeArray, N: 1, BitN: 1, Data: fromArray16([]uint16{lo})})
}
// If the container exists and bit is not set then update the page.
@ -155,7 +155,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
copy(other, a[:i])
other[i] = lo
copy(other[i+1:], a[i:])
return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), Data: fromArray16(other)})
return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), BitN: cell.BitN + 1, Data: fromArray16(other)})
case ContainerTypeRLE:
runs := toInterval16(cell.Data)
@ -163,7 +163,7 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
copy(c.rle[:], runs)
run, added := runAdd(c.rle[:len(runs)], lo)
if added {
leaf := checkRun(run, cell.Key)
leaf := checkRun(run, cell.BitN, cell.Key)
return true, c.putLeafCell(leaf)
}
return false, nil
@ -185,6 +185,8 @@ func (c *Cursor) Add(v uint64) (changed bool, err error) {
if err := c.tx.writeBitmapPage(pgno, fromArray64(a)); err != nil {
return false, err
}
// TODO(bbj): Update parent cell with new BitN.
return true, nil
default:
return false, fmt.Errorf("rbf.Cursor.Add(): invalid container type: %d", cell.Type)
@ -220,7 +222,7 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) {
other := make([]uint16, len(a)-1)
copy(other[:i], a[:i])
copy(other[i:], a[i+1:])
return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), Data: fromArray16(other)})
return true, c.putLeafCell(leafCell{Key: cell.Key, Type: ContainerTypeArray, N: len(other), BitN: cell.BitN - 1, Data: fromArray16(other)})
case ContainerTypeRLE:
r := toInterval16(cell.Data)
@ -265,6 +267,8 @@ func (c *Cursor) Remove(v uint64) (changed bool, err error) {
if err := c.tx.writeBitmapPage(pgno, fromArray64(a)); err != nil {
return false, err
}
// TODO(bbj): Update parent cell to decrement BitN.
return true, nil
default:
return false, fmt.Errorf("rbf.Cursor.Add(): invalid container type: %d", cell.Type)
@ -356,7 +360,10 @@ func (c *Cursor) putLeafCell(in leafCell) (err error) {
}
in.Data = fromArray64(a)
cell.Type = ContainerTypeBitmapPtr
bitmapPgno, _ := c.tx.allocate()
bitmapPgno, err := c.tx.allocate()
if err != nil {
return err
}
cell.Data = fromPgno(bitmapPgno)
}
@ -812,6 +819,7 @@ func (c *Cursor) Seek(key uint64) (exact bool, err error) {
if err != nil {
return false, err
}
switch typ := readFlags(buf); typ {
case PageTypeBranch:
n := readCellN(buf)
@ -1089,6 +1097,7 @@ func (c *Cursor) goNextPage() error {
func ConvertToLeafArgs(key uint64, c *roaring.Container) (result leafCell) {
result.Key = key
result.N = int(c.N())
result.BitN = int(c.N())
result.Type = ContainerTypeNone
if c.N() == 0 {
return
@ -1136,7 +1145,7 @@ func (c *Cursor) merge(key uint64, data *roaring.Container) (bool, error) {
if err != nil {
return false, errors.Wrap(err, "cursor.merge")
}
container = roaring.NewContainerBitmap(cell.N, d)
container = roaring.NewContainerBitmap(cell.BitN, d)
case ContainerTypeRLE:
d := toInterval16(cell.Data)
container = roaring.NewContainerRun(d)

47
rbf/cursor_test.go
View file

@ -31,7 +31,7 @@ func TestCursor_FirstNext(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
@ -95,7 +95,7 @@ func TestCursor_FirstNext_Quick(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
// Insert values in random order.
if err := tx.CreateBitmap("x"); err != nil {
@ -153,7 +153,7 @@ func TestCursor_LastPrev(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
@ -217,7 +217,7 @@ func TestCursor_LastPrev_Quick(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
// Insert values in random order.
if err := tx.CreateBitmap("x"); err != nil {
@ -276,7 +276,7 @@ func TestCursor_Union(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
@ -322,7 +322,7 @@ func TestCursor_Union(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
values := GenerateValues(rand, 10000)
rows := ToRows(values)
@ -356,7 +356,7 @@ func TestCursor_Intersect(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
row := make([]uint64, rbf.ShardWidth/64)
@ -403,7 +403,7 @@ func TestCursor_Intersect(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
values := GenerateValues(rand, rand.Intn(10000))
rows := ToRows(values)
@ -447,7 +447,7 @@ func TestCursor_AddRoaring(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
@ -466,7 +466,7 @@ func TestCursor_AddRoaring(t *testing.T) {
return bm
}(),
wantChanged: false,
wantErr: true},
wantErr: false},
{
name: "initial Array",
fieldview: "x",
@ -614,7 +614,7 @@ func TestCursor_RLETesting(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
//setup RLE
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
@ -696,10 +696,10 @@ func TestCursor_RLETesting(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
changed, err := tx.Add("x", tt.args...)
changeCount, err := tx.Add("x", tt.args...)
if tt.wantErr && err == nil {
t.Errorf("No Error %v", err)
} else if tt.wantChanged && !changed {
} else if tt.wantChanged && changeCount == 0 {
t.Errorf("No Change %v", err)
} else if err != nil {
t.Fatal(err)
@ -734,7 +734,7 @@ func TestCursor_RLEConversion(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
//setup RLE with full container
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
@ -840,7 +840,7 @@ func TestCursor_UpdateBranchCells(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
@ -914,7 +914,7 @@ func TestCursor_SplitBranchCells(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
@ -964,7 +964,7 @@ func TestCursor_RemoveCells(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
@ -1006,7 +1006,7 @@ func TestCursor_PlayContainer(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
@ -1035,14 +1035,13 @@ func TestCursor_PlayContainer(t *testing.T) {
if err := cur.First(); err != nil {
panic(err)
}
cur.Dump("fun.dot")
}
func TestCursor_OneBitmap(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
@ -1071,13 +1070,12 @@ func TestCursor_OneBitmap(t *testing.T) {
if err := cur.First(); err != nil {
panic(err)
}
cur.Dump("fun.dot")
}
func TestCursor_GenerateAll(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
@ -1111,9 +1109,4 @@ func TestCursor_GenerateAll(t *testing.T) {
if _, err := tx.AddRoaring("field/view/", bb); err != nil {
panic(err)
}
cur, err := tx.Cursor("field/view/")
if err != nil {
panic(err)
}
cur.Dump("fun.dot")
}

5
rbf/cursorx.go
View file

@ -93,10 +93,11 @@ func (c *Cursor) Dump(name string) {
fmt.Fprintf(bufStdout, "\n}")
bufStdout.Flush()
}
func (c *Cursor) Row(rowID uint64) (*roaring.Bitmap, error) {
func (c *Cursor) Row(shard, rowID uint64) (*roaring.Bitmap, error) {
base := rowID * ShardWidth
offset := uint64(c.tx.db.Shard * ShardWidth)
offset := uint64(shard * ShardWidth)
off := highbits(offset)
hi0, hi1 := highbits(base), highbits((rowID+1)*ShardWidth)
c.stack.index = 0

11
rbf/db.go
View file

@ -54,20 +54,15 @@ type DB struct {
// The maximum allowed database size. Required by mmap.
MaxSize int64
Shard int
}
// NewDB returns a new instance of DB.
func NewDB(path string) *DB {
return NewDBWithShard(path, 0)
}
func NewDBWithShard(path string, shard int) *DB {
return &DB{
txs: make(map[*Tx]struct{}),
pageMap: immutable.NewMap(&uint32Hasher{}),
Path: path,
MaxSize: DefaultMaxSize,
Shard: shard,
}
}
@ -97,7 +92,7 @@ func (db *DB) Open() (err error) {
db.mu.Lock()
defer db.mu.Unlock()
if err := os.MkdirAll(filepath.Dir(db.Path), 0755); err != nil {
if err := os.MkdirAll(db.Path, 0755); err != nil {
return err
} else if db.file, err = os.OpenFile(db.DataPath(), os.O_WRONLY|os.O_CREATE, 0666); err != nil {
return fmt.Errorf("open file: %w", err)
@ -569,7 +564,7 @@ func (db *DB) Begin(writable bool) (_ *Tx, err error) {
// This page is only written at the end of a dirty transaction.
page, err := db.readPage(db.pageMap, 0)
if err != nil {
_ = tx.Rollback()
tx.Rollback()
return nil, err
}
copy(tx.meta[:], page)
@ -613,7 +608,7 @@ func (db *DB) Check() error {
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
defer tx.Rollback()
return tx.Check()
}

5
rbf/db_test.go
View file

@ -119,14 +119,13 @@ func TestDB_Recovery(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer MustRollback(t, tx)
defer tx.Rollback()
if exists, err := tx.Contains("x", uint64(len(a))); exists || err != nil {
t.Fatalf("Contains()=<%v,%#v>", exists, err)
} else if exists, err := tx.Contains("x", uint64(len(a)-1)); !exists || err != nil {
t.Fatalf("Contains()=<%v,%#v>", exists, err)
} else if err := tx.Rollback(); err != nil {
t.Fatal(err)
}
tx.Rollback()
})
}

86
rbf/rbf.go
View file

@ -21,8 +21,11 @@ import (
"errors"
"fmt"
"io"
"math"
"os"
"unsafe"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/shardwidth"
)
@ -81,6 +84,8 @@ var (
ErrTxClosed = errors.New("transaction closed")
ErrTxNotWritable = errors.New("transaction not writable")
ErrBitmapNameRequired = errors.New("bitmap name required")
ErrBitmapNotFound = errors.New("bitmap not found")
ErrBitmapExists = errors.New("bitmap already exists")
)
// Debug is just a temporary flag used for debugging.
@ -257,8 +262,15 @@ func align8(offset int) int {
// leafCell represents a leaf cell.
type leafCell struct {
Key uint64
Type int
N int
Type int // container type
// N is the number of "things" in Data:
// for an array container the number of integers in the array.
// for an RLE, number of intervals.
// etc.
N int
BitN int
Data []byte
}
@ -361,19 +373,70 @@ func (c *leafCell) firstValue() uint16 {
}
}
// lastValue the last value from the container.
func (c *leafCell) lastValue() uint16 {
switch c.Type {
case ContainerTypeArray:
a := toArray16(c.Data)
return a[len(a)-1]
case ContainerTypeRLE:
r := toInterval16(c.Data)
return r[len(r)-1].Last
case ContainerTypeBitmap:
a := toArray64(c.Data)
for i := len(a) - 1; i >= 0; i-- {
for j := 63; j >= 0; j-- {
if a[i]&(1<<j) != 0 {
return (uint16(i) * 64) + uint16(j)
}
}
}
panic(fmt.Sprintf("rbf.leafCell.firstValue(): no values set in bitmap container: key=%d", c.Key))
default:
panic(fmt.Sprintf("invalid container type: %d", c.Type))
}
}
// countRange returns the bit count within the given range.
// We have to take int32 rather than uint16 because the interval is [start, end),
// and otherwise we have no way to ask to count the entire container (the
// high bit will be missed).
func (c *leafCell) countRange(start, end int32) (n int) {
// If the full range is being queried, simply use the precalculated count.
if start == 0 && end > math.MaxUint16 {
return c.BitN
}
switch c.Type {
case ContainerTypeArray:
return int(roaring.ArrayCountRange(toArray16(c.Data), start, end))
case ContainerTypeRLE:
return int(roaring.RunCountRange(toInterval16(c.Data), start, end))
case ContainerTypeBitmap:
return int(roaring.BitmapCountRange(toArray64(c.Data), start, end))
default:
panic(fmt.Sprintf("invalid container type: %d", c.Type))
}
}
func readLeafCellKey(page []byte, i int) uint64 {
offset := readCellOffset(page, i)
assert(offset < len(page))
return *(*uint64)(unsafe.Pointer(&page[offset]))
}
func readLeafCell(page []byte, i int) leafCell {
offset := readCellOffset(page, i)
// cd ..; PILOSA_TXSRC=rbf go test -v -run TestFragment_TopN_IDs -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0"
// gives panic: runtime error: slice bounds out of range [16390:8192] here.
buf := page[offset:]
var cell leafCell
cell.Key = *(*uint64)(unsafe.Pointer(&buf[0]))
cell.Type = int(*(*uint32)(unsafe.Pointer(&buf[8])))
cell.N = int(*(*uint32)(unsafe.Pointer(&buf[12])))
cell.N = int(*(*uint16)(unsafe.Pointer(&buf[12])))
cell.BitN = int(*(*uint16)(unsafe.Pointer(&buf[14])))
switch cell.Type {
case ContainerTypeArray:
@ -410,7 +473,8 @@ func writeLeafCell(page []byte, i, offset int, cell leafCell) {
writeCellOffset(page, i, offset)
*(*uint64)(unsafe.Pointer(&page[offset])) = cell.Key
*(*uint32)(unsafe.Pointer(&page[offset+8])) = uint32(cell.Type)
*(*uint32)(unsafe.Pointer(&page[offset+12])) = uint32(cell.N)
*(*uint16)(unsafe.Pointer(&page[offset+12])) = uint16(cell.N)
*(*uint16)(unsafe.Pointer(&page[offset+14])) = uint16(cell.BitN)
assert(offset+16+len(cell.Data) <= PageSize)
copy(page[offset+16:], cell.Data)
}
@ -486,8 +550,11 @@ func search(n int, f func(int) int) (index int, exact bool) {
return i, false
}
/*
func pagedumpi(b []byte, indent string, writer io.Writer) {
func Pagedump(b []byte, indent string, writer io.Writer) {
if writer == nil {
writer = os.Stderr
}
pgno := readPageNo(b)
if pgno == Magic32() {
fmt.Fprintf(writer, "==META\n")
@ -501,6 +568,7 @@ func pagedumpi(b []byte, indent string, writer io.Writer) {
// the page alone so this will output !PAGE for bitmap pages & invalid pages.
switch {
case flags&PageTypeLeaf != 0:
fmt.Fprintf(writer, "==LEAF pgno=%d flags=%d n=%d\n", pgno, flags, cellN)
for i := 0; i < cellN; i++ {
cell := readLeafCell(b, i)
switch cell.Type {
@ -525,7 +593,6 @@ func pagedumpi(b []byte, indent string, writer io.Writer) {
fmt.Fprintf(writer, "==!PAGE %d flags=%d\n", pgno, flags)
}
}
*/
func Walk(tx *Tx, pgno uint32, v func(uint32, []*RootRecord)) {
for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; {
@ -563,3 +630,8 @@ func RowValues(b []uint64) []uint64 {
}
return a
}
// func caller(skip int) string {
// _, file, line, _ := runtime.Caller(skip + 1)
// return fmt.Sprintf("%s:%d", file, line)
// }

8
rbf/rbf_test.go
View file

@ -119,14 +119,6 @@ func MustBegin(tb testing.TB, db *rbf.DB, writable bool) *rbf.Tx {
return tx
}
// MustRollback rolls back a transaction or fails.
func MustRollback(tb testing.TB, tx *rbf.Tx) {
tb.Helper()
if err := tx.Rollback(); err != nil && err != rbf.ErrTxClosed {
tb.Logf("rollback error: %q", err)
}
}
// MustAddRandom adds values to a bitmap in a random order.
func MustAddRandom(tb testing.TB, rand *rand.Rand, tx *rbf.Tx, name string, values ...uint64) {
tb.Helper()

958
rbf/tx.go
View file

File diff suppressed because it is too large Load diff

70
rbf/tx_test.go
View file

@ -29,13 +29,13 @@ func TestTx_CommitRollback(t *testing.T) {
defer MustCloseDB(t, db)
// Create bitmap in transaction but rollback.
if tx, err := db.Begin(true); err != nil {
tx, err := db.Begin(true)
if err != nil {
t.Fatal(err)
} else if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
} else if err := tx.Rollback(); err != nil {
t.Fatal(err)
}
tx.Rollback()
// Create bitmap in transaction again but commit.
if tx, err := db.Begin(true); err != nil {
@ -49,8 +49,8 @@ func TestTx_CommitRollback(t *testing.T) {
// Create bitmap again but it should fail as it already exists.
if tx, err := db.Begin(true); err != nil {
t.Fatal(err)
} else if err := tx.CreateBitmap("x"); err == nil || err.Error() != `bitmap already exists: "x"` {
_ = tx.Rollback()
} else if err := tx.CreateBitmap("x"); err == nil || err != rbf.ErrBitmapExists {
tx.Rollback()
t.Fatal(err)
} else if err := tx.Commit(); err != nil {
t.Fatal(err)
@ -62,13 +62,13 @@ func TestTx_CommitRollback(t *testing.T) {
defer func() { MustCloseDB(t, db) }()
// Create bitmap in transaction but rollback.
if tx, err := db.Begin(true); err != nil {
tx, err := db.Begin(true)
if err != nil {
t.Fatal(err)
} else if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
} else if err := tx.Rollback(); err != nil {
t.Fatal(err)
}
tx.Rollback()
db = MustReopenDB(t, db)
// Create bitmap in transaction again but commit.
@ -84,8 +84,8 @@ func TestTx_CommitRollback(t *testing.T) {
// Create bitmap again but it should fail as it already exists.
if tx, err := db.Begin(true); err != nil {
t.Fatal(err)
} else if err := tx.CreateBitmap("x"); err == nil || err.Error() != `bitmap already exists: "x"` {
_ = tx.Rollback()
} else if err := tx.CreateBitmap("x"); err == nil || err != rbf.ErrBitmapExists {
tx.Rollback()
t.Fatal(err)
} else if err := tx.Commit(); err != nil {
t.Fatal(err)
@ -102,7 +102,7 @@ func TestTx_CommitRollback(t *testing.T) {
tx0 := MustBegin(t, db, true)
go func() {
<-ch0
_ = tx0.Rollback()
tx0.Rollback()
}()
// Start separate write transaction in different goroutine.
@ -134,7 +134,7 @@ func TestTx_Add(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
@ -167,7 +167,7 @@ func TestTx_DeleteBitmap(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
// Create bitmap & add value.
if err := tx.CreateBitmap("x"); err != nil {
@ -192,7 +192,7 @@ func TestTx_RenameBitmap(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
// Create bitmap & add value.
if err := tx.CreateBitmap("x"); err != nil {
@ -226,7 +226,7 @@ func TestTx_Add_Quick(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
values := GenerateValues(rand, 10000)
if err := tx.CreateBitmap("x"); err != nil {
@ -265,7 +265,7 @@ func TestTx_AddRemove_Quick(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
values := GenerateValues(rand, 10000)
if err := tx.CreateBitmap("x"); err != nil {
@ -314,7 +314,7 @@ func TestTx_Multiple_CreateBitmap(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
values := GenerateValues(rand, 2)
if err := tx.CreateBitmap("x/1"); err != nil {
@ -332,7 +332,7 @@ func TestTx_Multiple_CreateBitmap(t *testing.T) {
}
tx1 := MustBegin(t, db, true)
defer func() { _ = tx1.Rollback() }()
defer tx1.Rollback()
if err := tx1.CreateBitmap("x/2"); err != nil {
t.Fatal(err)
@ -353,7 +353,7 @@ func TestTx_CursorCrashArray(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
@ -379,7 +379,7 @@ func TestTx_CursorCrashBitmap(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer MustRollback(t, tx)
defer tx.Rollback()
if err := tx.CreateBitmap("x"); err != nil {
t.Fatal(err)
}
@ -418,7 +418,7 @@ func BenchmarkTx_Add(b *testing.B) {
db := MustOpenDB(b)
defer MustCloseDB(b, db)
tx := MustBegin(b, db, true)
defer MustRollback(b, tx)
defer tx.Rollback()
for _, v := range values {
if _, err := tx.Add("x", v); err != nil {
@ -446,7 +446,7 @@ func BenchmarkTx_Contains(b *testing.B) {
db := MustOpenDB(b)
defer MustCloseDB(b, db)
tx := MustBegin(b, db, true)
defer MustRollback(b, tx)
defer tx.Rollback()
b.ResetTimer()
t := time.Now()
@ -464,3 +464,29 @@ func BenchmarkTx_Contains(b *testing.B) {
})
}
}
func TestTx_Dump(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(15)
nm := rbfName(field, view, shard)
if err := tx.CreateBitmap(nm); err != nil {
t.Fatal(err)
} else if _, err := tx.Add(nm, 0x00000001, 0x00000002, 0x00010003, 0x00030004); err != nil {
t.Fatal(err)
}
// test that we don't crash, and get *something* back
s := tx.DumpString(index)
if s == "" {
panic("should have had 3 containers!")
}
}
func rbfName(field, view string, shard uint64) string {
return fmt.Sprintf("%s\x00%s\x00%d", field, view, shard)
}

169
rbf/vprint.go Normal file
View file

@ -0,0 +1,169 @@
// home: https://github.com/glyerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package rbf
import (
"fmt"
"io"
"os"
"path"
"runtime"
"runtime/debug"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("\n%s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}
var _ = stack // happy linter

6
roaring/container_stash.go
View file

@ -387,7 +387,7 @@ func (c *Container) bitmap() []uint64 {
// is provided. The target should be zeroed, or this becomes an implicit
// union.
func (c *Container) AsBitmap(target []uint64) (out []uint64) {
if c.typeID == containerBitmap {
if c != nil && c.typeID == containerBitmap {
return c.bitmap()
}
// Reminder: len(nil) == 0.
@ -399,6 +399,10 @@ func (c *Container) AsBitmap(target []uint64) (out []uint64) {
out[i] = 0
}
}
// A nil *Container is a valid empty container.
if c == nil {
return out
}
if c.typeID == containerArray {
a := c.array()
for _, v := range a {

31
roaring/roaring.go
View file

@ -2860,20 +2860,19 @@ func (c *Container) countRange(start, end int32) (n int32) {
return 0
}
if c.isArray() {
return c.arrayCountRange(start, end)
return ArrayCountRange(c.array(), start, end)
} else if c.isRun() {
return c.runCountRange(start, end)
return RunCountRange(c.runs(), start, end)
}
return c.bitmapCountRange(start, end)
return BitmapCountRange(c.bitmap(), start, end)
}
func (c *Container) arrayCountRange(start, end int32) (n int32) {
func ArrayCountRange(array []uint16, start, end int32) (n int32) {
if roaringParanoia {
if start > end {
panic(fmt.Sprintf("counting in range but %v > %v", start, end))
}
}
array := c.array()
i := int32(sort.Search(len(array), func(i int) bool { return int32(array[i]) >= start }))
for ; i < int32(len(array)); i++ {
v := int32(array[i])
@ -2885,7 +2884,7 @@ func (c *Container) arrayCountRange(start, end int32) (n int32) {
return n
}
func (c *Container) bitmapCountRange(start, end int32) int32 {
func BitmapCountRange(bitmap []uint64, start, end int32) int32 {
if roaringParanoia {
if start > end {
panic(fmt.Sprintf("counting in range but %v > %v", start, end))
@ -2894,7 +2893,6 @@ func (c *Container) bitmapCountRange(start, end int32) int32 {
var n uint64
i, j := start/64, end/64
// Special case when start and end fall in the same word.
bitmap := c.bitmap()
if i == j {
offi, offj := uint(start%64), uint(64-end%64)
n += popcount((bitmap[i] >> offi) << (offj + offi))
@ -2921,13 +2919,13 @@ func (c *Container) bitmapCountRange(start, end int32) int32 {
return int32(n)
}
func (c *Container) runCountRange(start, end int32) (n int32) {
// RunCountRange returns the ranged bit count for RLE pairs.
func RunCountRange(runs []Interval16, start, end int32) (n int32) {
if roaringParanoia {
if start > end {
panic(fmt.Sprintf("counting in range but %v > %v", start, end))
}
}
runs := c.runs()
for _, iv := range runs {
// iv is before range
if int32(iv.Last) < start {
@ -3837,12 +3835,12 @@ func (c *Container) check() error {
a.Append(fmt.Errorf("array count mismatch: count=%d, n=%d", len(array), c.N()))
}
} else if c.isRun() {
n := c.runCountRange(0, MaxContainerVal+1)
n := RunCountRange(c.runs(), 0, MaxContainerVal+1)
if n != c.N() {
a.Append(fmt.Errorf("run count mismatch: count=%d, n=%d", n, c.N()))
}
} else if c.isBitmap() {
if n := c.bitmapCountRange(0, MaxContainerVal+1); n != c.N() {
if n := BitmapCountRange(c.bitmap(), 0, MaxContainerVal+1); n != c.N() {
a.Append(fmt.Errorf("bitmap count mismatch: count=%d, n=%d", n, c.N()))
}
} else {
@ -4052,7 +4050,7 @@ func intersectionCountRunRun(a, b *Container) (n int32) {
func intersectionCountBitmapRun(a, b *Container) (n int32) {
statsHit("intersectionCount/BitmapRun")
for _, iv := range b.runs() {
n += a.bitmapCountRange(int32(iv.Start), int32(iv.Last)+1)
n += BitmapCountRange(a.bitmap(), int32(iv.Start), int32(iv.Last)+1)
}
return n
}
@ -4370,6 +4368,7 @@ func unionArrayArray(a, b *Container) *Container {
break
}
}
// note: len(output) CAN be > 4096
return NewContainerArray(output)
}
@ -6867,8 +6866,12 @@ func ConvertRunToBitmap(c *Container) {
func Optimize(c *Container) {
c.optimize()
}
func Union(a, b *Container) *Container {
return union(a, b)
func Union(a, b *Container) (c *Container) {
c = union(a, b)
// c can be have arrays that are too big, and need
// to be optimized into raw bitmaps.
c.optimize()
return c
}
func Difference(a, b *Container) *Container {

67
roaring/roaring_internal_test.go
View file

@ -131,7 +131,7 @@ func TestContainerRunAdd2(t *testing.T) {
func TestRunCountRange(t *testing.T) {
c := NewContainerRun(nil)
cnt := c.runCountRange(2, 9)
cnt := RunCountRange(c.runs(), 2, 9)
if cnt != 0 {
t.Fatalf("should get 0 from empty container, but got: %v", cnt)
}
@ -139,7 +139,7 @@ func TestRunCountRange(t *testing.T) {
c.add(6)
c.add(7)
cnt = c.runCountRange(2, 9)
cnt = RunCountRange(c.runs(), 2, 9)
if cnt != 3 {
t.Fatalf("should get 3 from interval within range, but got: %v", cnt)
}
@ -149,52 +149,52 @@ func TestRunCountRange(t *testing.T) {
c.add(10)
c.add(11)
cnt = c.runCountRange(4, 8)
cnt = RunCountRange(c.runs(), 4, 8)
if cnt != 3 {
t.Fatalf("should get 3 from range overlaps front of interval, but got: %v", cnt)
}
cnt = c.runCountRange(5, 8)
cnt = RunCountRange(c.runs(), 5, 8)
if cnt != 3 {
t.Fatalf("should get 3 from range within interval, but got: %v", cnt)
}
cnt = c.runCountRange(6, 8)
cnt = RunCountRange(c.runs(), 6, 8)
if cnt != 2 {
t.Fatalf("should get 2 from range within interval, but got: %v", cnt)
}
cnt = c.runCountRange(3, 9)
cnt = RunCountRange(c.runs(), 3, 9)
if cnt != 4 {
t.Fatalf("should get 4 from range overlaps front of interval, but got: %v", cnt)
}
cnt = c.runCountRange(9, 14)
cnt = RunCountRange(c.runs(), 9, 14)
if cnt != 3 {
t.Fatalf("should get 3 from range overlaps back of interval, but got: %v", cnt)
}
cnt = c.runCountRange(8, 10)
cnt = RunCountRange(c.runs(), 8, 10)
if cnt != 2 {
t.Fatalf("should get 2 from range within interval, but got: %v", cnt)
}
cnt = c.runCountRange(8, 11)
cnt = RunCountRange(c.runs(), 8, 11)
if cnt != 3 {
t.Fatalf("should get 3 from range within interval, but got: %v", cnt)
}
cnt = c.runCountRange(8, 12)
cnt = RunCountRange(c.runs(), 8, 12)
if cnt != 4 {
t.Fatalf("should get 4 from range overlaps back of interval, but got: %v", cnt)
}
cnt = c.runCountRange(5, 12)
cnt = RunCountRange(c.runs(), 5, 12)
if cnt != 7 {
t.Fatalf("should get 7 from interval within range, but got: %v", cnt)
}
cnt = c.runCountRange(5, 11)
cnt = RunCountRange(c.runs(), 5, 11)
if cnt != 6 {
t.Fatalf("should get 6 from interval equal to range, but got: %v", cnt)
}
@ -203,7 +203,7 @@ func TestRunCountRange(t *testing.T) {
c.add(19)
c.add(18)
cnt = c.runCountRange(1, 22)
cnt = RunCountRange(c.runs(), 1, 22)
if cnt != 10 {
t.Fatalf("should get 10 from multiple ranges in interval, but got: %v", cnt)
}
@ -211,7 +211,7 @@ func TestRunCountRange(t *testing.T) {
c.add(13)
c.add(14)
cnt = c.runCountRange(6, 18)
cnt = RunCountRange(c.runs(), 6, 18)
if cnt != 9 {
t.Fatalf("should get 9 from multiple ranges overlapping both sides, but got: %v", cnt)
}
@ -263,7 +263,7 @@ func TestBitmapCountRange(t *testing.T) {
for i, test := range tests {
c.setBitmap(test.bitmap[:])
if ret := c.bitmapCountRange(test.start, test.end); ret != test.exp {
if ret := BitmapCountRange(c.bitmap(), test.start, test.end); ret != test.exp {
t.Fatalf("test #%v count of %v from %v to %v should be %v but got %v", i, test.bitmap, test.start, test.end, test.exp, ret)
}
}
@ -4404,3 +4404,40 @@ func TestUnionRunRunInPlaceBitwiseCompare(t *testing.T) {
}
}
}
func TestCloneRoaringIterator(t *testing.T) {
ca := NewContainerArray([]uint16{1, 10, 100, 1000})
ba := NewFileBitmap()
ba.Containers.Put(0, ca)
ba.Containers.Put(10, ca)
ba.Containers.Put(101, ca)
ba.Containers.Put(10001, ca)
var buf bytes.Buffer
_, err := ba.WriteTo(&buf)
if err != nil {
t.Fatalf("error writing: %v", err)
}
itr, err := NewRoaringIterator(buf.Bytes())
if err != nil {
t.Fatalf("error NewRoaringIterator(buf.Bytes()): %v", err)
}
itr2 := itr.Clone()
var keys []uint64
for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() {
keys = append(keys, itrKey)
_ = synthC
}
var keys2 []uint64
for itrKey, synthC := itr2.NextContainer(); synthC != nil; itrKey, synthC = itr2.NextContainer() {
keys2 = append(keys2, itrKey)
_ = synthC
}
if !reflect.DeepEqual(keys, keys2) {
t.Fatalf("keys != keys2. keys='%#v'; keys2='%#v'", keys, keys2)
}
}

118
row.go
View file

@ -18,7 +18,6 @@ import (
"encoding/json"
"sort"
"github.com/molecula/ext"
pb "github.com/pilosa/pilosa/v2/proto"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
@ -326,65 +325,6 @@ func (r *Row) Union(others ...*Row) *Row {
return &Row{segments: output}
}
// GenericBinaryOp returns the output of a generic op on r and other.
func (r *Row) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *Row, args map[string]interface{}) *Row {
var segments []rowSegment
itr := newMergeSegmentIterator(r.segments, other.segments)
for s0, s1 := itr.next(); s0 != nil || s1 != nil; s0, s1 = itr.next() {
if s1 == nil {
segments = append(segments, *s0)
continue
} else if s0 == nil {
segments = append(segments, *s1)
continue
}
segments = append(segments, *s0.GenericBinaryOp(op, s1, args))
}
return &Row{segments: segments}
}
// GenericNaryOp returns the output of an nary op on r and others.
func (r *Row) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*Row, args map[string]interface{}) *Row {
segments := make([][]rowSegment, 0, len(others)+1)
if len(r.segments) > 0 {
segments = append(segments, r.segments)
}
nextSegs := make([][]rowSegment, 0, len(others)+1)
toProcess := make([]*rowSegment, 0, len(others)+1)
var output []rowSegment
for _, other := range others {
if len(other.segments) > 0 {
segments = append(segments, other.segments)
}
}
for len(segments) > 0 {
shard := segments[0][0].shard
for _, segs := range segments {
if segs[0].shard < shard {
shard = segs[0].shard
}
}
nextSegs = nextSegs[:0]
toProcess := toProcess[:0]
for _, segs := range segments {
if segs[0].shard == shard {
toProcess = append(toProcess, &segs[0])
segs = segs[1:]
}
if len(segs) > 0 {
nextSegs = append(nextSegs, segs)
}
}
// at this point, "toProcess" is a list of all the segments
// sharing the lowest ID, and nextSegs is a list of all the others.
// Swap the segment lists (so we don't have to reallocate it)
segments, nextSegs = nextSegs, segments
output = append(output, *toProcess[0].GenericNaryOp(op, toProcess[1:], args))
}
return &Row{segments: output}
}
// Difference returns the diff of r and other.
func (r *Row) Difference(others ...*Row) *Row {
var output []rowSegment
@ -408,17 +348,6 @@ func (r *Row) Difference(others ...*Row) *Row {
return &Row{segments: output}
}
// GenericUnaryOp returns the results of a generic op on r.
func (r *Row) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *Row {
work := r
var segments []rowSegment
for _, segment := range work.segments {
opped := segment.GenericUnaryOp(op, args)
segments = append(segments, *opped)
}
return &Row{segments: segments}
}
// Shift returns the bitwise shift of r by n bits.
// Currently only positive shift values are supported.
//
@ -523,15 +452,6 @@ func (r *Row) Count() uint64 {
return n
}
// GenericCount applies an op to lots of things.
func (r *Row) GenericCount(op ext.BitmapOpUnaryCount, args map[string]interface{}) uint64 {
var n int64
for i := range r.segments {
n += op([]ext.Bitmap{WrapBitmap(r.segments[i].data)}, args)
}
return uint64(n)
}
// MarshalJSON returns a JSON-encoded byte slice of r.
func (r *Row) MarshalJSON() ([]byte, error) {
var o struct {
@ -644,33 +564,6 @@ func (s *rowSegment) Union(others ...*rowSegment) *rowSegment {
}
}
// GenericOp performs a generic op on s and other
func (s *rowSegment) GenericBinaryOp(op ext.GenericBitmapOpBitmap, other *rowSegment, args map[string]interface{}) *rowSegment {
data := op([]ext.Bitmap{WrapBitmap(s.data), WrapBitmap(other.data)}, args)
return &rowSegment{
data: UnwrapBitmap(data),
shard: s.shard,
n: data.Count(),
}
}
// GenericOp performs a generic op on s and others
func (s *rowSegment) GenericNaryOp(op ext.GenericBitmapOpBitmap, others []*rowSegment, args map[string]interface{}) *rowSegment {
bitmaps := make([]ext.Bitmap, len(others)+1)
bitmaps[0] = WrapBitmap(s.data)
for i, seg := range others {
bitmaps[i+1] = WrapBitmap(seg.data)
}
data := op(bitmaps, args)
return &rowSegment{
data: UnwrapBitmap(data),
shard: s.shard,
n: data.Count(),
}
}
// Difference returns the diff of s and other.
func (s *rowSegment) Difference(others ...*rowSegment) *rowSegment {
datas := make([]*roaring.Bitmap, len(others))
@ -713,17 +606,6 @@ func (s *rowSegment) Shift() (*rowSegment, error) {
}, nil
}
// GenericUnaryOp returns s subject to op.
func (s *rowSegment) GenericUnaryOp(op ext.GenericBitmapOpBitmap, args map[string]interface{}) *rowSegment {
data := UnwrapBitmap(op([]ext.Bitmap{WrapBitmap(s.data)}, args))
return &rowSegment{
data: data,
shard: s.shard,
n: data.Count(),
}
}
// SetBit sets the i-th column of the row.
func (s *rowSegment) SetBit(i uint64) (changed bool) {
s.ensureWritable()

86
server.go
View file

@ -27,13 +27,11 @@ import (
"sync"
"time"
"github.com/molecula/ext"
uuid "github.com/satori/go.uuid"
// extensions pulls in some extensions depending on build tags
_ "github.com/pilosa/pilosa/v2/extensions"
"github.com/pilosa/pilosa/v2/logger"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
"github.com/pkg/errors"
@ -63,7 +61,6 @@ type Server struct { // nolint: maligned
hosts []string
clusterDisabled bool
serializer Serializer
extensions []*ext.ExtensionInfo
// External
systemInfo SystemInfo
@ -430,30 +427,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.cluster.confirmDownRetries = s.confirmDownRetries
s.cluster.confirmDownSleep = s.confirmDownSleep
s.holder.broadcaster = s
err = s.loadAllExtensions()
if err != nil {
s.logger.Printf("not all plugins loaded successfully")
}
if len(s.extensions) > 0 {
s.logger.Printf("loaded extensions:")
for _, ext := range s.extensions {
if ext == nil {
s.logger.Printf(" inexplicably, a nil extension?!?")
continue
}
s.logger.Printf(" %s %s: %s", ext.Name, ext.Version, ext.Description)
if ext.License != "" {
s.logger.Printf(" License: %s", ext.License)
}
if len(ext.BitmapOps) > 0 {
opList := make([]string, len(ext.BitmapOps))
for i := range ext.BitmapOps {
opList[i] = ext.BitmapOps[i].Name
}
s.logger.Printf(" Ops: %s", strings.Join(opList, ", "))
}
}
}
err = s.cluster.setup()
if err != nil {
@ -467,56 +440,6 @@ func (s *Server) InternalClient() InternalClient {
return s.defaultClient
}
// loadNewExtensions loads extensions that have been
// registered since the last call to loadNewExtensions.
func (s *Server) loadNewExtensions() error { //nolint:unused
return s.loadExtensions(ext.NewExtensions())
}
// loadAllExtensions loads all extensions.
func (s *Server) loadAllExtensions() error {
return s.loadExtensions(ext.AllExtensions())
}
func (s *Server) loadExtensions(exts []*ext.ExtensionInfo) error {
var lastError error
for _, extension := range exts {
if err := s.loadExtension(extension); err != nil {
lastError = err
}
}
return lastError
}
func (s *Server) loadExtension(extInfo *ext.ExtensionInfo) error {
if extInfo.ExtensionAPI != "v0" {
return fmt.Errorf("%s: unsupported extension API %s", extInfo.Name, extInfo.ExtensionAPI)
}
s.extensions = append(s.extensions, extInfo)
bitmapOps := extInfo.BitmapOps
bmOps, countOps, fieldOps, unknownOps := 0, 0, 0, 0
for i := range bitmapOps {
typ := bitmapOps[i].Func.BitmapOpType()
switch {
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputCount:
countOps++
case typ.Input == ext.OpInputBitmap && typ.Output == ext.OpOutputBitmap:
bmOps++
case typ.Input == ext.OpInputNaryBSI && typ.Output == ext.OpOutputSignedBitmap:
fieldOps++
default:
unknownOps++
}
}
err := s.executor.registerOps(bitmapOps)
if err != nil {
s.logger.Printf("warning: extension registration failed: %v", err)
} else {
pql.RegisterPluginFuncs(bitmapOps)
}
return nil
}
// UpAndDown brings the server up minimally and shuts it down
// again; basically, it exists for testing holder open and close.
func (s *Server) UpAndDown() error {
@ -545,8 +468,12 @@ func (s *Server) UpAndDown() error {
func (s *Server) Open() error {
s.logger.Printf("open server")
// Start background monitoring.
s.snapshotQueue = newSnapshotQueue(10, 2, s.logger)
if s.holder.NeedsSnapshot() {
// Start background monitoring.
s.snapshotQueue = newSnapshotQueue(10, 2, s.logger)
} else {
s.snapshotQueue = defaultSnapshotQueue //TODO (twg) rethink this
}
// Log startup
err := s.holder.logStartup()
@ -711,6 +638,7 @@ func (s *Server) monitorAntiEntropy() {
// the cluster sets its state to resizing and *then* sends to
// abortAntiEntropyCh before starting to resize
}
// Sync holders.
s.logger.Printf("holder sync beginning")
s.cluster.muAntiEntropy.Lock()

54
server/cluster_test.go
View file

@ -143,9 +143,9 @@ func TestClusterResize_AddNode(t *testing.T) {
clus := test.MustRunCluster(t, 2)
defer clus.Close()
if !checkClusterState(clus[0], pilosa.ClusterStateNormal, 1000) {
if !test.CheckClusterState(clus[0], pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", clus[0].API.State())
} else if !checkClusterState(clus[1], pilosa.ClusterStateNormal, 1000) {
} else if !test.CheckClusterState(clus[1], pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", clus[1].API.State())
}
})
@ -176,9 +176,9 @@ func TestClusterResize_AddNode(t *testing.T) {
}
defer m1.Close()
if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
}
})
@ -224,9 +224,9 @@ func TestClusterResize_AddNode(t *testing.T) {
}
defer m1.Close()
if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
}
@ -273,9 +273,9 @@ func TestClusterResize_AddNode(t *testing.T) {
}
defer m1.Close()
if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
}
@ -326,9 +326,9 @@ func TestClusterResize_AddNode(t *testing.T) {
}
defer m1.Close()
if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
}
@ -373,9 +373,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
}
defer m1.Close()
if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
}
@ -431,9 +431,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
}()
defer m1.Close()
if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
}
@ -489,9 +489,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
}
defer m1.Close()
if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
}
@ -545,9 +545,9 @@ func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
}
defer m1.Close()
if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
}
m0.QueryExpect(t, "i", "", `Row(f=1)`, exp)
@ -598,11 +598,11 @@ func TestCluster_GossipMembership(t *testing.T) {
t.Fatal(err)
}
if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) {
if !test.CheckClusterState(m0, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", m0.API.State())
} else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) {
} else if !test.CheckClusterState(m1, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", m1.API.State())
} else if !checkClusterState(m2, pilosa.ClusterStateNormal, 1000) {
} else if !test.CheckClusterState(m2, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node2 cluster state: %s", m2.API.State())
}
@ -725,15 +725,3 @@ func TestClusterMutualTLS(t *testing.T) {
t.Fatal(err)
}
}
// checkClusterState polls a given cluster for its state until it
// receives a matching state. It polls up to n times before returning.
func checkClusterState(m *test.Command, state string, n int) bool {
for i := 0; i < n; i++ {
if m.API.State() == state {
return true
}
time.Sleep(10 * time.Millisecond)
}
return false
}

12
test/cluster.go
View file

@ -193,6 +193,18 @@ func MustNewCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Clu
return c
}
// CheckClusterState polls a given cluster for its state until it
// receives a matching state. It polls up to n times before returning.
func CheckClusterState(m *Command, state string, n int) bool {
for i := 0; i < n; i++ {
if m.API.State() == state {
return true
}
time.Sleep(10 * time.Millisecond)
}
return false
}
// newCluster creates a new cluster
func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (Cluster, error) {
if size == 0 {

12
test/index.go
View file

@ -32,7 +32,9 @@ func newIndex() *Index {
if err != nil {
panic(err)
}
index, err := pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, "i")
h := pilosa.NewHolder(pilosa.DefaultPartitionN)
h.Path = path
index, err := h.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
panic(err)
}
@ -42,7 +44,7 @@ func newIndex() *Index {
// MustOpenIndex returns a new, opened index at a temporary path. Panic on error.
func MustOpenIndex() *Index {
index := newIndex()
if err := index.Open(); err != nil {
if err := index.Open(false); err != nil {
panic(err)
}
return index
@ -62,12 +64,14 @@ func (i *Index) Reopen() error {
}
path, name := i.Path(), i.Name()
i.Index, err = pilosa.NewIndex(pilosa.NewHolder(pilosa.DefaultPartitionN), path, name)
h := pilosa.NewHolder(pilosa.DefaultPartitionN)
h.Path = h.HolderPathFromIndexPath(path, name)
i.Index, err = h.CreateIndex(name, pilosa.IndexOptions{})
if err != nil {
return err
}
if err := i.Open(); err != nil {
if err := i.Open(false); err != nil {
return err
}
return nil

76
translator_test.go
View file

@ -296,6 +296,82 @@ func TestTranslation_Reset(t *testing.T) {
})
}
// Test index key translation replication under node failure.
func TestTranslation_Replication(t *testing.T) {
t.Run("Replication", func(t *testing.T) {
c := test.MustRunCluster(t, 3,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerIsCoordinator(true),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerReplicaN(2),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerIsCoordinator(false),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerReplicaN(2),
)},
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerIsCoordinator(false),
pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
pilosa.OptServerReplicaN(2),
)},
)
node0 := c[0]
node1 := c[1]
ctx := context.Background()
idx := "i"
field := "f"
// Create an index with keys.
if _, err := node0.API.CreateIndex(ctx, idx,
pilosa.IndexOptions{
Keys: true,
}); err != nil {
t.Fatal(err)
}
if _, err := node0.API.CreateField(ctx, idx, field); err != nil {
t.Fatal(err)
}
// Write data on first node.
// these keys are a minimal example to reproduce the problem for the case of a 3-node cluster with replication factor 2
if _, err := node0.Queryf(t, idx, "", `
Set("x1", f=1)
Set("x2", f=1)
`); err != nil {
t.Fatal(err)
}
exp := `{"results":[{"attrs":{},"columns":[],"keys":["x1","x2"]}]}`
if !test.CheckClusterState(node0, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node0 cluster state: %s", node0.API.State())
} else if !test.CheckClusterState(node1, pilosa.ClusterStateNormal, 1000) {
t.Fatalf("unexpected node1 cluster state: %s", node1.API.State())
}
// Verify the data exists
node0.QueryExpect(t, idx, "", `Row(f=1)`, exp)
// Kill one node.
if err := node1.Command.Close(); err != nil {
t.Fatal(err)
}
// Verify the data exists with one node down
node0.QueryExpect(t, idx, "", `Row(f=1)`, exp)
})
}
// Test key translation with multiple nodes.
func TestTranslation_Coordinator(t *testing.T) {
// Ensure that field key translations requests sent to

397
tx.go
View file

@ -15,9 +15,16 @@
package pilosa
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"github.com/pilosa/pilosa/v2/rbf"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
@ -50,6 +57,10 @@ const writable = true
// that have not been committed.
type Tx interface {
// Type returns "roaring", "rbf", "badger", "badger_roaring", or one of the other
// blue-green Tx types at the top of txfactory.go
Type() string
// Rollback must be called the end of read-only transactions. Either
// Rollback or Commit must be called at the end of writable transactions.
// It is safe to call Rollback multiple times, but it must be
@ -97,6 +108,11 @@ type Tx interface {
// Calling Next() on the returned roaring.ContainerIterator gives
// you a roaring.Container that is either run, array, or raw bitmap.
// Return value 'found' is true when the ckey container was present.
// ckey of 0 gives all containers (in the fragment).
//
// ContainerIterator must not have side-effects. blueGreenTx will
// call it at the very beginning of commit to verify db contents.
//
ContainerIterator(index, field, view string, shard uint64, ckey uint64) (citer roaring.ContainerIterator, found bool, err error)
// RoaringBitmap retreives the roaring.Bitmap for the entire shard.
@ -162,10 +178,57 @@ type Tx interface {
OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error)
// ImportRoaringBits does efficient bulk import using rit, a roaring.RoaringIterator.
//
// See the roaring package for details of the RoaringIterator.
//
// If clear is true, the bits from rit are cleared, otherwise they are set in the
// specifed fragment.
ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error)
//
// The data argument can be nil, its ignored for RBF/BadgerTx. It is supplied to
// RoaringTx.ImportRoaringBits() in fragment.go fragment.fillFragmentFromArchive()
// to do the traditional fragment.readStorageFromArchive() which
// does some in memory field/view/fragment metadata updates.
// It makes blueGreenTx testing viable too.
//
// ImportRoaringBits return values changed and rowSet may be inaccurate if
// the data []byte is supplied (the RoaringTx implementation neglects this for speed).
ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error)
RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error)
// SliceOfShards returns all of the shards for the specified index, field, view triple.
// Use within pilosa supposes a new read-only transaction was created just
// for the SliceOfShards() call. The legacy RoaringTx version is the only
// one that needs optionalViewPath; any other Tx implementation can ignore that.
SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error)
// Dump is for debugging, what does this Tx see as its database?
Dump()
}
// TxStore has operations that will create and commit multiple
// Tx on a backing store.
type TxStore interface {
// DeleteFragment deletes all the containers in a fragment.
//
// This is not in a Tx because it will often do too many deletes for a single
// transaction, and clients would be suprised to find their Tx had already
// been commited and they are getting an error on double-Commit.
// Instead each TxStore implementation creates and commits as many
// transactions as needed.
//
// Argument frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
// If not nil, it must be of type *fragment. If frag is supplied, then
// index must be equal to frag.index, field equal to frag.field, view equal
// to frag.view, and shard equal to frag.shard.
//
DeleteFragment(index, field, view string, shard uint64, frag interface{}) error
DeleteField(index, field string) error
// Close shuts down the database.
Close() error
}
// RawRoaringData used by ImportRoaringBits.
@ -207,10 +270,39 @@ func NewMultiTxWithIndex(writable bool, index *Index) *MultiTx {
var _ Tx = (*MultiTx)(nil)
func (mtx *MultiTx) Type() string {
return RoaringTxn
}
// debugging, what does this Tx see as its database?
func (mtx *MultiTx) Dump() {
mtx.mu.Lock()
defer mtx.mu.Unlock()
if len(mtx.txs) == 0 {
return
}
for _, tx := range mtx.txs {
tx.Dump()
return
}
}
func (mtx *MultiTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
tx, err := mtx.txNoShard(index)
panicOn(err)
return tx.SliceOfShards(index, field, view, optionalViewPath)
}
func (mtx *MultiTx) UseRowCache() bool {
return true
}
func (mtx *MultiTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
}
func (mtx *MultiTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
tx, err := mtx.tx(index, shard)
panicOn(err)
@ -226,8 +318,10 @@ func (mtx *MultiTx) Pointer() string {
return fmt.Sprintf("%p", mtx)
}
func (tx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
panic("not done")
func (mtx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
}
func (mtx *MultiTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
@ -412,6 +506,20 @@ func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) {
return tx, nil
}
// version of the above for SliceOfShards(), where we don't have a shard.
func (mtx *MultiTx) txNoShard(index string) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
// Lookup transaction from cache.
for _, tx := range mtx.txs {
if tx.(*RoaringTx).Index.name == index {
return tx, nil
}
}
panic(fmt.Sprintf("no prior RoaringTx available, looking up index='%v'", index))
}
type multiTxKey struct {
index string
shard uint64
@ -426,10 +534,51 @@ type RoaringTx struct {
fragment *fragment
}
func (tx *RoaringTx) Type() string {
return RoaringTxn
}
func (tx *RoaringTx) Dump() {
fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys())
}
func (tx *RoaringTx) UseRowCache() bool {
return true
}
func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
// SliceOfShards is based on view.openFragments()
file, err := os.Open(filepath.Join(optionalViewPath, "fragments"))
if os.IsNotExist(err) {
return
} else if err != nil {
return nil, errors.Wrap(err, "opening fragments directory")
}
defer file.Close()
fis, err := file.Readdir(0)
if err != nil {
return nil, errors.Wrap(err, "reading fragments directory")
}
for _, fi := range fis {
if fi.IsDir() {
continue
}
// Parse filename into integer.
shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
//AlwaysPrintf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", index, field, view, fi.Name())
//v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name())
continue
}
sliceOfShards = append(sliceOfShards, shard)
}
return
}
func (tx *RoaringTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
@ -442,13 +591,24 @@ func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roa
return b.Iterator()
}
func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
b, err := tx.bitmap(index, field, view, shard)
panicOn(err)
// ImportRoaringBits return values changed and rowSet will be inaccurate if
// the data []byte is supplied. This mimics the traditional roaring-per-file
// and should be faster.
func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
f, err := tx.getFragment(index, field, view, shard)
if err != nil {
return 0, nil, err
}
return b.ImportRoaringRawIterator(rit, clear, true, rowSize)
if len(data) > 0 {
// changed and rowSet are ignored anyway when len(data) > 0;
// when we are called from fragment.fillFragmentFromArchive()
// which is the only place the data []byte is supplied.
// blueGreenTx also turns off the checks in this case.
return 0, nil, f.readStorageFromArchive(bytes.NewBuffer(data))
}
changed, rowSet, err = f.storage.ImportRoaringRawIterator(rit, clear, true, rowSize)
return
}
func (tx *RoaringTx) Readonly() bool {
@ -625,8 +785,16 @@ func (tx *RoaringTx) OffsetRange(index, field, view string, shard uint64, offset
// getFragment is used by IncrementOpN() and by bitmap()
func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*fragment, error) {
// If a fragment is attached, always use it.
// If a fragment is attached, always use it. Since it was set at Tx creation,
// it is highly likely to be correct.
if tx.fragment != nil {
// but still a basic sanity check.
if tx.fragment.index != index ||
tx.fragment.field != field ||
tx.fragment.view != view ||
tx.fragment.shard != shard {
panic(fmt.Sprintf("different fragment cached vs requested. tx.fragment='%#v', index='%v', field='%v'; view='%v'; shard='%v'", tx.fragment, index, field, view, shard))
}
return tx.fragment, nil
}
@ -659,8 +827,10 @@ func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*frag
}
frag := v.Fragment(shard)
if frag == nil {
panic(fmt.Sprintf("fragment not found: %q / %q / %d", field, view, shard))
return nil, fmt.Errorf("fragment not found: %q / %q / %d", field, view, shard)
//panic(fmt.Sprintf("fragment not found: %q / %q / %d", field, view, shard))
}
// Note: we cannot cache frag into tx.fragment.
@ -677,3 +847,212 @@ func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.B
}
return frag.storage, nil
}
type RoaringStore struct{}
func NewRoaringStore() *RoaringStore {
return &RoaringStore{}
}
func (db *RoaringStore) Close() error {
return nil
}
func (db *RoaringStore) DeleteField(index, field, fieldPath string) error {
// under blue-green badger_roaring, the directory will not be found, b/c badger will have
// already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs:
// "If the path does not exist, RemoveAll returns nil (no error)"
err := os.RemoveAll(fieldPath)
if err != nil {
return errors.Wrap(err, "removing directory")
}
return nil
}
// frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
func (db *RoaringStore) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error {
fragment, ok := frag.(*fragment)
if !ok {
return fmt.Errorf("RoaringStore.DeleteFragment must get frag of type *fragment, but got '%T'", frag)
}
// Close data files before deletion.
if err := fragment.Close(); err != nil {
return errors.Wrap(err, "closing fragment")
}
// Delete fragment file.
if err := os.Remove(fragment.path); err != nil {
return errors.Wrap(err, "deleting fragment file")
}
// Delete fragment cache file.
if err := os.Remove(fragment.cachePath()); err != nil {
return errors.Wrap(err, fmt.Sprintf("no cache file to delete for shard %d", fragment.shard))
}
return nil
}
func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
file, err := os.Open(fragmentPathForRoaring) // open the fragment file
if err != nil {
return nil, -1, err
}
fi, err := file.Stat()
if err != nil {
return nil, -1, errors.Wrap(err, "statting")
}
sz = fi.Size()
r = file
return
}
type RBFTx struct {
index string
tx *rbf.Tx
}
func (tx *RBFTx) Type() string {
return RBFTxn
}
func (tx *RBFTx) Rollback() {
tx.tx.Rollback()
}
func (tx *RBFTx) Commit() error {
return tx.tx.Commit()
}
func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
return tx.tx.RoaringBitmap(rbfName(field, view, shard))
}
func (tx *RBFTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
return tx.tx.Container(rbfName(field, view, shard), key)
}
func (tx *RBFTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
return tx.tx.PutContainer(rbfName(field, view, shard), key, c)
}
func (tx *RBFTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
return tx.tx.RemoveContainer(rbfName(field, view, shard), key)
}
func (tx *RBFTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
return tx.tx.Add(rbfName(field, view, shard), a...)
}
func (tx *RBFTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
return tx.tx.Remove(rbfName(field, view, shard), a...)
}
func (tx *RBFTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
return tx.tx.Contains(rbfName(field, view, shard), v)
}
func (tx *RBFTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
return tx.tx.ContainerIterator(rbfName(field, view, shard), key)
}
func (tx *RBFTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
return tx.tx.ForEach(rbfName(field, view, shard), fn)
}
func (tx *RBFTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
return tx.tx.ForEachRange(rbfName(field, view, shard), start, end, fn)
}
func (tx *RBFTx) Count(index, field, view string, shard uint64) (uint64, error) {
return tx.tx.Count(rbfName(field, view, shard))
}
func (tx *RBFTx) Max(index, field, view string, shard uint64) (uint64, error) {
return tx.tx.Max(rbfName(field, view, shard))
}
func (tx *RBFTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
return tx.tx.Min(rbfName(field, view, shard))
}
func (tx *RBFTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
return tx.tx.UnionInPlace(rbfName(field, view, shard), others...)
}
func (tx *RBFTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
return tx.tx.CountRange(rbfName(field, view, shard), start, end)
}
func (tx *RBFTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
return tx.tx.OffsetRange(rbfName(field, view, shard), offset, start, end)
}
func (tx *RBFTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {}
func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
return tx.tx.ImportRoaringBits(rbfName(field, view, shard), rit, clear, log, rowSize, data)
}
func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
panic("TODO: Implement RBFTx.RoaringBitmapReader()")
}
func (tx *RBFTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) {
prefix := rbfFieldViewPrefix(field, view)
names, err := tx.tx.BitmapNames()
if err != nil {
return nil, err
}
// Iterate over shard names and collect shards from matching field/view prefix.
for _, name := range names {
if !strings.HasPrefix(name, prefix) {
continue
}
s := strings.TrimPrefix(name, prefix)
shard, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return nil, errors.Wrap(err, "parse shard id from rbf key")
}
sliceOfShards = append(sliceOfShards, shard)
}
return sliceOfShards, nil
}
func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
b, err := tx.RoaringBitmap(index, field, view, shard)
panicOn(err)
return b.Iterator()
}
func (tx *RBFTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
func (tx *RBFTx) Dump() {
tx.tx.Dump(tx.index)
}
// Readonly is true if the transaction is not read-and-write, but only doing reads.
func (tx *RBFTx) Readonly() bool {
return !tx.tx.Writable()
}
func (tx *RBFTx) UseRowCache() bool {
return false
}
// rbfName returns a NULL-separated key used for identifying bitmap maps in RBF.
func rbfName(field, view string, shard uint64) string {
return fmt.Sprintf("%s\x00%s\x00%d", field, view, shard)
}
// rbfFieldViewPrefix returns a NULL-separated prefix for keys in RBF.
func rbfFieldViewPrefix(field, view string) string {
return fmt.Sprintf("%s\x00%s\x00", field, view)
}

374
txfactory.go
View file

@ -21,7 +21,9 @@ import (
"strconv"
"strings"
"syscall"
"text/tabwriter"
"github.com/pilosa/pilosa/v2/rbf"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
@ -55,13 +57,15 @@ var sep = string(os.PathSeparator)
type TxFactory struct {
typeOfTx txtype
bw *BadgerDBWrapper
badgerDB *BadgerDBWrapper
rbfDB *rbf.DB
roaringDB *RoaringStore
// could have more than one *Index, but for now keep it simple,
// and allow blueGreenTx to report badger contents via idx
idx *Index
// TODO: put RBF database handle here.
}
// integer types for fast switch{}
@ -85,6 +89,32 @@ const (
blueGreenRBFBadger txtype = 9
)
func (txf *TxFactory) NeedsSnapshot() bool {
switch txf.typeOfTx {
case noneTxn:
panic("noneTxn should not occur")
case roaringFragmentFilesTxn:
return true
case badgerTxn:
return false
case rbfTxn:
return false
case blueGreenBadgerRoaring:
return true
case blueGreenRoaringBadger:
return true
case blueGreenRBFRoaring:
return true
case blueGreenRoaringRBF:
return true
case blueGreenBadgerRBF:
return false
case blueGreenRBFBadger:
return false
}
panic(fmt.Sprintf("unknown typeOfTx '%v'", txf.typeOfTx))
}
func MustTxsrcToTxtype(txsrc string) txtype {
switch txsrc {
case RoaringTxn: // "roaring"
@ -109,29 +139,55 @@ func MustTxsrcToTxtype(txsrc string) txtype {
panic(fmt.Sprintf("unknown txsrc '%v'", txsrc))
}
func newTxFactory(txsrc string, path string) (f *TxFactory, err error) {
// always store files in a subdir of dir. If we are having one
// database or many can depend on name.
func NewTxFactory(txsrc string, dir, name string, openExisting bool) (f *TxFactory, err error) {
ty := MustTxsrcToTxtype(txsrc)
if ty < 1 || ty > 9 {
panic(fmt.Sprintf("invalid txtype '%v'", int(ty)))
}
var bw *BadgerDBWrapper
if ty == badgerTxn || ty == 4 || ty == 5 || ty == 8 || ty == 9 {
bw, err = openBadgerDBWrapper(path)
// TODO(jea): figure out what the appropriate error path is here.
//fmt.Printf("warning: could not open badgerdb on path '%v': '%v'. For safety, we are opening a new '%v-fallback' instead\n", path, err, path+"-fallback")
if err != nil {
//bw, err = newBadgerDBWrapper(path + "-fallback")
bw, err = newBadgerDBWrapper(path)
}
panicOn(err)
bw.doAllocZero = true
f = &TxFactory{
typeOfTx: ty,
roaringDB: NewRoaringStore(),
}
return &TxFactory{
typeOfTx: ty,
bw: bw,
}, err
switch ty {
case badgerTxn, blueGreenBadgerRoaring, blueGreenRoaringBadger, blueGreenBadgerRBF, blueGreenRBFBadger:
// one, big, bad-ass badger for all data: the honeyBadger.
//
// Note that having a single Tx backing store for all indexes
// enables cross-index Tx, which are important and are tested for.
path := dir + sep + "honeyBadger"
if openExisting {
f.badgerDB, err = globalBadgerReg.openBadgerDBWrapper(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot open badger db. path='%v'", path))
}
} else {
f.badgerDB, err = globalBadgerReg.newBadgerDBWrapper(path)
if err != nil {
return nil, errors.Wrap(err, fmt.Sprintf("cannot create new badger db. path='%v'", path))
}
}
// electric-fence like finding of access to mmapped data beyond
// transaction end time.
f.badgerDB.doAllocZero = true
}
switch ty {
case rbfTxn, blueGreenRBFRoaring, blueGreenRoaringRBF, blueGreenBadgerRBF, blueGreenRBFBadger:
f.rbfDB = rbf.NewDB(filepath.Join(dir, "db.rbf"))
if err := f.rbfDB.Open(); err != nil {
return nil, errors.Wrap(err, "cannot open rbf db")
}
}
return f, err
}
// Txo holds the transaction options
@ -153,72 +209,151 @@ func (f *TxFactory) DeleteIndex(name string) error {
// from holder.go:955, by default is already done there with os.RemoveAll()
return nil
case badgerTxn:
return f.bw.DeleteIndex(name)
return f.badgerDB.DeleteIndex(name)
case rbfTxn:
panic("todo rbfTxn DeleteIndex(name)")
case blueGreenBadgerRoaring:
return f.bw.DeleteIndex(name)
return f.badgerDB.DeleteIndex(name)
case blueGreenRoaringBadger:
return f.bw.DeleteIndex(name)
return f.badgerDB.DeleteIndex(name)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
func (f *TxFactory) Close() error {
func (f *TxFactory) DeleteFieldFromStore(index, field, fieldPath string) error {
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return nil
return f.roaringDB.DeleteField(index, field, fieldPath)
case badgerTxn:
// note cannot actually close Badger here.
// causes problems b/c tries holder.DeleteIndex tries to delete the index after db is closed.
//return f.bw.Close()
return nil
return f.badgerDB.DeleteField(index, field, fieldPath)
case rbfTxn:
panic("todo rbfTxn Close()")
//return f.rbfDB.DeleteField(index, field, fieldPath)
return nil
case blueGreenBadgerRoaring:
return nil
_ = f.badgerDB.DeleteField(index, field, fieldPath)
return f.roaringDB.DeleteField(index, field, fieldPath)
case blueGreenRoaringBadger:
return nil
_ = f.roaringDB.DeleteField(index, field, fieldPath)
return f.badgerDB.DeleteField(index, field, fieldPath)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
func (f *TxFactory) DeleteFragmentFromStore(index, field, view string, shard uint64, frag *fragment) error {
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
case badgerTxn:
return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
case rbfTxn:
tx, err := f.rbfDB.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()
if err := tx.DeleteBitmapsWithPrefix(rbfFieldViewPrefix(field, view)); err != nil {
return err
}
return tx.Commit()
case blueGreenBadgerRoaring:
_ = f.badgerDB.DeleteFragment(index, field, view, shard, frag)
return f.roaringDB.DeleteFragment(index, field, view, shard, frag)
case blueGreenRoaringBadger:
_ = f.roaringDB.DeleteFragment(index, field, view, shard, frag)
return f.badgerDB.DeleteFragment(index, field, view, shard, frag)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
func (f *TxFactory) CloseIndex(idx *Index) error {
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return nil
case badgerTxn:
// note cannot actually close Badger here.
// causes problems b/c tries holder.DeleteIndex tries to delete the index after db is closed.
//return f.badgerDB.Close()
return nil
case rbfTxn:
panic("todo rbfTxn CloseIndex()")
return f.rbfDB.Close()
case blueGreenBadgerRoaring:
return nil
case blueGreenRoaringBadger:
return nil
case blueGreenRBFRoaring:
_ = f.rbfDB.Close()
return nil
case blueGreenRoaringRBF:
return f.rbfDB.Close()
case blueGreenBadgerRBF:
return f.rbfDB.Close()
case blueGreenRBFBadger:
_ = f.rbfDB.Close()
return nil
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
func (f *TxFactory) NewTx(o Txo) Tx {
indexName := ""
if o.Index != nil {
indexName = o.Index.name
}
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
case badgerTxn:
btx := f.bw.NewBadgerTx(o.Write)
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
return btx
case rbfTxn:
panic("todo rbfTxn creation")
tx, err := f.rbfDB.Begin(o.Write)
if err != nil {
panic(err) // TODO: Add error return on NewTx()
}
return &RBFTx{tx: tx, index: indexName}
case blueGreenBadgerRoaring:
btx := f.bw.NewBadgerTx(o.Write)
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(btx, rtx, f.idx)
case blueGreenRoaringBadger:
btx := f.bw.NewBadgerTx(o.Write)
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(rtx, btx, f.idx)
case blueGreenBadgerRBF:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
rbftx, err := f.rbfDB.Begin(o.Write)
if err != nil {
panic(errors.Wrap(err, "rbfDB.Begin transaction errored"))
}
return newBlueGreenTx(btx, &RBFTx{tx: rbftx, index: indexName}, f.idx)
case blueGreenRBFBadger:
btx := f.badgerDB.NewBadgerTx(o.Write, indexName)
rbftx, err := f.rbfDB.Begin(o.Write)
if err != nil {
panic(errors.Wrap(err, "rbfDB.Begin transaction errored"))
}
return newBlueGreenTx(&RBFTx{tx: rbftx, index: indexName}, btx, f.idx)
case blueGreenRBFRoaring:
rbftx, err := f.rbfDB.Begin(o.Write)
if err != nil {
panic(errors.Wrap(err, "rbfDB.Begin transaction errored"))
}
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(&RBFTx{tx: rbftx, index: indexName}, rtx, f.idx)
case blueGreenRoaringRBF:
rbftx, err := f.rbfDB.Begin(o.Write)
if err != nil {
panic(errors.Wrap(err, "rbfDB.Begin transaction errored"))
}
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(rtx, &RBFTx{tx: rbftx, index: indexName}, f.idx)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
@ -255,14 +390,13 @@ func (ty txtype) String() string {
// Hence to view uncommited keys, you must provide in optionalUseThisTx the
// Tx in which they have been added.
func (idx *Index) StringifiedBadgerKeys(optionalUseThisTx Tx) string {
return idx.Txf.bw.StringifiedBadgerKeys(optionalUseThisTx)
return idx.Txf.badgerDB.StringifiedBadgerKeys(optionalUseThisTx)
}
// fragmentSpecFromRoaringPath takes a path releative to the
// index directory, not including the name of the index itself.
// The path should not start with the path separator sep ('/' or '\\') rune.
func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64, err error) {
if len(path) == 0 {
err = fmt.Errorf("fragmentSpecFromRoaringPath error: path '%v' too short", path)
return
@ -291,29 +425,38 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64,
}
func (idx *Index) StringifiedRoaringKeys() (r string) {
paths, err := listFilesUnderDir(idx.path, false, "", true)
panicOn(err)
index := idx.name
r = "allkeys:[\n"
n := 0
for _, relpath := range paths {
field, view, shard, err := fragmentSpecFromRoaringPath(relpath)
if err != nil {
continue // ignore .meta paths
}
abspath := idx.path + sep + relpath
s, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard)
const showOps = false
s, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard, showOps)
panicOn(err)
//r += fmt.Sprintf("path:'%v' fragment contains:\n") + s
if s == "" {
s = "<empty bitmap>"
}
r += s
n++
}
if n == 0 {
return "" // new convention that empty database => empty string returned.
}
// note that we can have a bitmap present, but it can be empty
r += "]\n all-in-blake3:" + blake3sum16([]byte(r)) + "\n"
return "roaring-" + r
}
func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64) (r string, err error) {
func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64, showOps bool) (r string, err error) {
var info roaring.BitmapInfo
_ = info
@ -352,6 +495,21 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard
return
}
//cmd.DisplayInfo(info)
// inlined
if showOps {
pC := pointerContext{
from: info.From,
to: info.To,
}
if info.ContainerCount > 0 {
printContainers(info, pC)
}
if info.Ops > 0 {
printOps(info)
}
}
citer, found := rbm.Containers.Iterator(0)
_ = found // probably gonna use just the Ops log instead, so don't panic if !found.
@ -432,28 +590,6 @@ func fileSize(name string) (int64, error) {
var _ = fileSize // happy linter
// Dump prints to stdout the contents of the roaring Containers
// stored in idx. Its format may vary depending of the type of
// idx.Txf transaction factory that is in use.
// Mostly for debugging.
func (idx *Index) Dump(label string) {
ty := idx.Txf.TxType()
fileline := FileLine(2)
switch ty {
case badgerTxn:
fmt.Printf("%v Index.Dump('%v') for index '%v':\n%v\n", fileline, label, idx.name, idx.StringifiedBadgerKeys(nil))
return
case blueGreenRoaringBadger, blueGreenBadgerRoaring:
fmt.Printf("%v Index.Dump('%v') for index '%v', RoaringTx:\n%v\n", fileline, label, idx.name, idx.StringifiedRoaringKeys())
fmt.Printf("%v Index.Dump('%v') for index '%v', BadgerTx :\n%v\n", fileline, label, idx.name, idx.StringifiedBadgerKeys(nil))
return
case roaringFragmentFilesTxn:
fmt.Printf("%v Index.Dump('%v') for index '%v', BadgerTx :\n%v\n", fileline, label, idx.name, idx.StringifiedRoaringKeys())
return
}
panic(fmt.Errorf("%v Index.Dump('%v') for index '%v': no implementation for txtype '%v'\n", fileline, label, idx.name, ty))
}
func containerToBytes(ct *roaring.Container) []byte {
ty := roaring.ContainerType(ct)
switch ty {
@ -468,3 +604,109 @@ func containerToBytes(ct *roaring.Container) []byte {
}
panic(fmt.Sprintf("unknown container type '%v'", int(ty)))
}
type pointerContext struct {
from, to uintptr
}
func printOps(info roaring.BitmapInfo) {
fmt.Fprintln(os.Stdout, " Ops:")
tw := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
fmt.Fprintf(tw, " \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE")
printed := 0
for _, op := range info.OpDetails {
fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size)
printed++
}
tw.Flush()
}
func (p *pointerContext) pretty(c roaring.ContainerInfo) string {
var pointer string
if c.Mapped {
if c.Pointer >= p.from && c.Pointer < p.to {
pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from)
} else {
pointer = fmt.Sprintf("!0x%x!", c.Pointer)
}
} else {
pointer = fmt.Sprintf("0x%x", c.Pointer)
}
return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer)
}
// stolen from ctl/inspect.go
func printContainers(info roaring.BitmapInfo, pC pointerContext) {
fmt.Fprintln(os.Stdout, " Containers:")
tw := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0)
fmt.Fprintf(tw, " \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n")
fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS")
c1s := info.Containers
c2s := info.OpContainers
l1 := len(c1s)
l2 := len(c2s)
i1 := 0
i2 := 0
var c1, c2 roaring.ContainerInfo
c1.Key = ^uint64(0)
c2.Key = ^uint64(0)
c1e := false
c2e := false
if i1 < l1 {
c1 = c1s[i1]
i1++
c1e = true
}
if i2 < l2 {
c2 = c2s[i2]
i2++
c2e = true
}
printed := 0
for c1e || c2e {
c1used := false
c2used := false
var key uint64
c1fmt := "-\t\t\t"
c2fmt := "-\t\t\t"
// If c2 exists, we'll always prefer its flags,
// if it doesn't, this gets overwritten.
flags := c2.Flags
if !c2e || (c1e && c1.Key < c2.Key) {
c1fmt = pC.pretty(c1)
key = c1.Key
c1used = true
flags = c1.Flags
} else if !c1e || (c2e && c2.Key < c1.Key) {
c2fmt = pC.pretty(c2)
key = c2.Key
c2used = true
} else {
// c1e and c2e both set, and neither key is < the other.
c1fmt = pC.pretty(c1)
c2fmt = pC.pretty(c2)
key = c1.Key
c1used = true
c2used = true
}
if c1used {
if i1 < l1 {
c1 = c1s[i1]
i1++
} else {
c1e = false
}
}
if c2used {
if i2 < l2 {
c2 = c2s[i2]
i2++
} else {
c2e = false
}
}
fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags)
printed++
}
tw.Flush()
}

42
utils_internal_test.go
View file

@ -15,8 +15,6 @@
package pilosa
import (
"bufio"
"bytes"
"fmt"
"io/ioutil"
"path/filepath"
@ -396,9 +394,8 @@ func (b bcast) SendTo(to *Node, m Message) error {
return nil
}
// FollowResizeInstruction is a version of cluster.FollowResizeInstruction used for testing.
// FollowResizeInstruction is a version of cluster.followResizeInstruction used for testing.
func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error {
// Prepare the return message.
complete := &ResizeInstructionComplete{
JobID: instr.JobID,
@ -420,7 +417,8 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
}
// Sync available shards.
for _, is := range instr.NodeStatus.Indexes {
for k, is := range instr.NodeStatus.Indexes {
_ = k
for _, fs := range is.Fields {
f := destCluster.holder.Field(is.Name, fs.Name)
@ -451,23 +449,29 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error
}
}
buf := bytes.NewBuffer(nil)
// this is the *test* version of a network call, transferring fragments between
// nodes in a cluster. So it is allowed to be kind of a hack.
bw := bufio.NewWriter(buf)
br := bufio.NewReader(buf)
// there will be two -badgerdb directories/databases, we need to copy
// from src to dest the fragment. This simulates sending the fragment over the network.
srcIdx := srcCluster.holder.Index(src.Index)
srctx := srcIdx.Txf.NewTx(Txo{Write: !writable, Index: srcIdx, Fragment: srcFragment})
// Get the fragment from source.
if _, err := srcFragment.WriteTo(bw); err != nil {
return err
}
// Flush the bufio.buf to the io.Writer (buf).
bw.Flush()
// Write data to destination.
if _, err := destFragment.ReadFrom(br); err != nil {
return err
destIdx := destCluster.holder.Index(src.Index)
desttx := destIdx.Txf.NewTx(Txo{Write: writable, Index: destIdx, Fragment: destFragment})
citer, _, err := srctx.ContainerIterator(src.Index, src.Field, src.View, src.Shard, 0)
panicOn(err)
d := destFragment
for citer.Next() {
ckey, c := citer.Value()
err := desttx.PutContainer(d.index, d.field, d.view, d.shard, ckey, c)
panicOn(err)
}
citer.Close()
panicOn(desttx.Commit())
srctx.Rollback()
}
return nil

93
view.go
View file

@ -50,6 +50,7 @@ type view struct {
qualifiedName string
holder *Holder
idx *Index
fieldType string
cacheType string
@ -136,14 +137,18 @@ func (v *view) open() error {
if err := func() error {
// Ensure the view's path exists.
v.holder.Logger.Debugf("ensure view path exists: %s", v.path)
if err := os.MkdirAll(v.path, 0777); err != nil {
err := os.MkdirAll(v.path, 0777)
if err != nil {
return errors.Wrap(err, "creating view directory")
} else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil {
}
err = os.MkdirAll(filepath.Join(v.path, "fragments"), 0777)
if err != nil {
return errors.Wrap(err, "creating fragments directory")
}
v.holder.Logger.Debugf("open fragments for index/field/view: %s/%s/%s", v.index, v.field, v.name)
if err := v.openFragments(); err != nil {
if err := v.openFragmentsInTx(); err != nil {
return errors.Wrap(err, "opening fragments")
}
@ -159,49 +164,46 @@ func (v *view) open() error {
var workQueue = make(chan struct{}, runtime.NumCPU()*2)
// openFragments opens and initializes the fragments inside the view.
func (v *view) openFragments() error {
file, err := os.Open(filepath.Join(v.path, "fragments"))
if os.IsNotExist(err) {
return nil
} else if err != nil {
return errors.Wrap(err, "opening fragments directory")
}
defer file.Close()
// replaces v.openFragments() with Tx generic code.
func (v *view) openFragmentsInTx() error {
fis, err := file.Readdir(0)
tx := v.idx.Txf.NewTx(Txo{Write: !writable, Index: v.idx})
defer tx.Rollback()
shards, err := tx.SliceOfShards(v.index, v.field, v.name, v.path)
if err != nil {
return errors.Wrap(err, "reading fragments directory")
return errors.Wrap(err, "SliceOfShards")
}
eg, ctx := errgroup.WithContext(context.Background())
var mu sync.Mutex
fileLoop:
for _, loopFi := range fis {
shardCh := make(chan uint64, len(shards))
for i := range shards {
shardCh <- shards[i]
}
shardLoop:
for range shards {
select {
case <-ctx.Done():
break fileLoop
break shardLoop
default:
fi := loopFi
if fi.IsDir() {
continue
}
// Parse filename into integer.
shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
if err != nil {
v.holder.Logger.Debugf("WARNING: couldn't use non-integer file as shard in index/field/view %s/%s/%s: %s", v.index, v.field, v.name, fi.Name())
continue
}
workQueue <- struct{}{}
v.holder.Logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard)
eg.Go(func() error {
defer func() {
<-workQueue
}()
var shard uint64
select {
case shard = <-shardCh:
default:
return nil // no more work
}
v.holder.Logger.Debugf("open index/field/view/fragment: %s/%s/%s/%d", v.index, v.field, v.name, shard)
frag := v.newFragment(v.fragmentPath(shard), shard)
if err := frag.Open(); err != nil {
return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)
@ -359,6 +361,16 @@ func (v *view) notifyIfNewShard(shard uint64) {
}
func (v *view) newFragment(path string, shard uint64) *fragment {
if v.holder != nil && v.idx != nil {
// A view must have its v.idx *Index registered with its holder.
// Otherwise TestField_AvailableShards crashes, as one example.
hIdx := v.holder.Index(v.idx.name)
if hIdx == nil && v.idx != nil {
v.holder.addIndexFromField(v.idx)
}
}
frag := newFragment(v.holder, path, v.index, v.field, v.name, shard, v.flags())
frag.CacheType = v.cacheType
frag.CacheSize = v.cacheSize
@ -375,28 +387,17 @@ func (v *view) newFragment(path string, shard uint64) *fragment {
func (v *view) deleteFragment(shard uint64) error {
v.mu.Lock()
defer v.mu.Unlock()
fragment := v.fragments[shard]
if fragment == nil {
f := v.fragments[shard]
if f == nil {
return ErrFragmentNotFound
}
v.holder.Logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard)
// Close data files before deletion.
if err := fragment.Close(); err != nil {
return errors.Wrap(err, "closing fragment")
idx := f.holder.Index(v.index)
if err := idx.Txf.DeleteFragmentFromStore(f.index, f.field, f.view, f.shard, f); err != nil {
return errors.Wrap(err, "DeleteFragment")
}
// Delete fragment file.
if err := os.Remove(fragment.path); err != nil {
return errors.Wrap(err, "deleting fragment file")
}
// Delete fragment cache file.
if err := os.Remove(fragment.cachePath()); err != nil {
v.holder.Logger.Printf("no cache file to delete for shard %d", shard)
}
delete(v.fragments, shard)
v.removeKnownShard(shard)

10
view_internal_test.go
View file

@ -34,7 +34,15 @@ func mustOpenView(index, field, name string) *view {
CacheSize: DefaultCacheSize,
}
v := newView(NewHolder(DefaultPartitionN), path, index, field, name, fo)
h := NewHolder(DefaultPartitionN)
h.Path = path
// h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment
idx, err := h.createIndex(index, IndexOptions{})
_ = idx
panicOn(err)
v := newView(h, path, index, field, name, fo)
v.idx = idx
if err := v.open(); err != nil {
panic(err)
}

54
vprint.go
View file

@ -111,3 +111,57 @@ func FileLine(depth int) string {
func stack() string {
return string(debug.Stack())
}
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}

2
xrbrsupport.go
View file

@ -63,7 +63,7 @@ func (rbc *RBFConverter) Convert(index, field, view string, shard uint64, rb *ro
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
defer tx.Rollback()
name := fmt.Sprintf("%s/%s", field, view)
err = tx.CreateBitmap(name)