scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-10 15:01:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #565 from molecula/tx_roaring_badger

Browse source 
integration of Tx, RoaringTx and BadgerTx implementations.
This commit is contained in:
jaten-molecula 2020-07-20 16:01:45 -04:00 • committed by GitHub
commit 1c9ef3d321
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 6265 additions and 424 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
Makefile
View file

@ -149,6 +149,28 @@ docker-build:
docker-test:
docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS)' $(TESTFLAGS) ./...
# run top tests, not subdirs. print summary red/green after.
# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
topt:
go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.roar
@echo " log.topt green: \c"; cat log.topt.roar | grep PASS |wc -l
@echo " log.topt red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l
topt-badger:
PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger
@echo " log.topt green: \c"; cat log.topt.badger | grep PASS |wc -l
@echo " log.topt red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l
topt-rbf:
PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.rbf
@echo " log.topt green: \c"; cat log.topt.rbf | grep PASS |wc -l
@echo " log.topt red: \c"; cat log.topt.rbf | grep '\-\-\- FAIL' |wc -l
topt-race:
go test -race -v -tags='$(BUILD_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.race
@echo " log.topt green: \c"; cat log.topt.race | grep PASS |wc -l
@echo " log.topt red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l
# Run golangci-lint
golangci-lint: require-golangci-lint
golangci-lint run --skip-files '.*\.peg\.go'

43
api.go
View file

@ -330,6 +330,7 @@ func setUpImportOptions(opts ...ImportOption) (*ImportOptions, error) {
type importJob struct {
ctx context.Context
tx Tx
req *ImportRoaringRequest
shard uint64
field *Field
@ -370,17 +371,23 @@ func importWorker(importWork chan importJob) {
var doClear bool
switch doAction {
case RequestActionOverwrite:
tx := &RoaringTx{Field: j.field}
// TODO(jea): the question here is, why are we commiting this separately from j.tx?
// why doesn't j.tx suffice? It doesn't but why/which is correct?
tx := j.field.holder.indexes[j.field.index].Txf.NewTx(Txo{Write: true, Field: j.field})
defer tx.Rollback()
if err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block); err != nil {
return errors.Wrap(err, "importing roaring as overwrite")
}
if err := tx.Commit(); err != nil {
return errors.Wrap(err, "commit of importing roaring as overwrite")
}
case RequestActionClear:
doClear = true
fallthrough
case RequestActionSet:
fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2]))
if fileMagic == roaring.MagicNumber { // if pilosa roaring format
if err := j.field.importRoaring(j.ctx, viewData, j.shard, viewName, doClear); err != nil {
if err := j.field.importRoaring(j.ctx, j.tx, viewData, j.shard, viewName, doClear); err != nil {
return errors.Wrap(err, "importing pilosa roaring")
}
} else {
@ -388,7 +395,7 @@ func importWorker(importWork chan importJob) {
// field.importRoaring changes the standard roaring run format to pilosa roaring
data := make([]byte, len(viewData))
copy(data, viewData)
if err := j.field.importRoaring(j.ctx, data, j.shard, viewName, doClear); err != nil {
if err := j.field.importRoaring(j.ctx, j.tx, data, j.shard, viewName, doClear); err != nil {
return errors.Wrap(err, "importing standard roaring")
}
}
@ -439,6 +446,10 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
return newPreconditionFailedError(err)
}
// Obtain transaction.
tx := index.Txf.NewTx(Txo{Write: true, Index: index})
defer tx.Rollback()
nodes := api.cluster.shardNodes(indexName, shard)
errCh := make(chan error, len(nodes))
for _, node := range nodes {
@ -446,6 +457,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
if node.ID == api.server.nodeID {
api.importWork <- importJob{
ctx: ctx,
tx: tx,
req: req,
shard: shard,
field: field,
@ -465,9 +477,11 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
for {
select {
case <-ctx.Done():
// defered tx.Rollback() happens automatically here.
return ctx.Err()
case nodeErr := <-errCh:
if nodeErr != nil {
// defered tx.Rollback() happens automatically here.
return nodeErr
}
maxNode++
@ -475,7 +489,7 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
// Exit once all nodes are processed.
if maxNode == len(nodes) {
return nil
return tx.Commit()
}
}
}
@ -583,7 +597,8 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
}
// Obtain transaction
tx := &RoaringTx{Index: index}
tx := index.Txf.NewTx(Txo{Write: !writable, Index: index})
defer tx.Rollback()
// Wrap writer with a CSV writer.
cw := csv.NewWriter(w)
@ -626,10 +641,8 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, fieldName strin
// Ensure data is flushed.
cw.Flush()
span.LogKV("n", n)
return nil
return tx.Commit()
}
// ShardNodes returns the node and all replicas which should contain a shard's data.
@ -1061,7 +1074,8 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
}
// Obtain transaction.
tx := &RoaringTx{Index: index}
tx := index.Txf.NewTx(Txo{Write: true, Index: index})
defer tx.Rollback()
if err := req.ValidateWithTimestamp(index.CreatedAt(), field.CreatedAt()); err != nil {
return errors.Wrap(err, "validating import value request")
@ -1165,8 +1179,12 @@ func (api *API) Import(ctx context.Context, req *ImportRequest, opts ...ImportOp
err = field.Import(tx, req.RowIDs, req.ColumnIDs, timestamps, opts...)
if err != nil {
api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
} else {
err = tx.Commit()
}
return errors.Wrap(err, "importing")
}
// ImportValue bulk imports values into a particular field.
@ -1188,7 +1206,8 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
}
// Obtain transaction.
tx := &RoaringTx{Index: index}
tx := index.Txf.NewTx(Txo{Write: true, Index: index})
defer tx.Rollback()
// Set up import options.
options, err := setUpImportOptions(opts...)
@ -1274,7 +1293,9 @@ func (api *API) ImportValue(ctx context.Context, req *ImportValueRequest, opts .
api.server.logger.Printf("import error: index=%s, field=%s, shard=%d, columns=%d, err=%s", req.Index, req.Field, req.Shard, len(req.ColumnIDs), err)
}
}
if err == nil {
err = tx.Commit()
}
return errors.Wrap(err, "importing value")
}

1701
badger.go Normal file
View file

File diff suppressed because it is too large Load diff

1442
badger_test.go Normal file
View file

File diff suppressed because it is too large Load diff

95
blake3.go Normal file
View file

@ -0,0 +1,95 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"encoding/binary"
"fmt"
"sync"
cryptorand "crypto/rand"
"github.com/zeebo/blake3"
)
// Blake3Hasher is a thread/goroutine safe way to
// obtain a blake3 cryptographic hash of input []byte.
// Reference https://github.com/BLAKE3-team/BLAKE3
// suggests it is 6x faster than BLAKE2B.
// The Go github.com/zeebo/blake3 version is
// AVX2 and SSE4.1 accelerated.
type Blake3Hasher struct {
hasher *blake3.Hasher
hasherMu sync.Mutex
}
// NewBlake3Hasher returns a new Blake3Hasher.
func NewBlake3Hasher() *Blake3Hasher {
return &Blake3Hasher{
hasher: blake3.New(),
}
}
// CryptoHash writes the blake3 cryptographic hash of
// input into buffer and returns it.
// Like the standard libary's hash.Hash interface's Sum() method,
// the buffer is re-used and overwritten
// to avoid allocation. The caller determines the byte length of
// the outputCryptohash by the size of the supplied buffer
// slice, and this will be exactly equal to the supplies bytes.
// In this way, shorter or longer hashes can be provided as
// needed.
func (w *Blake3Hasher) CryptoHash(input []byte, buffer []byte) (outputCryptohash []byte) {
w.hasherMu.Lock()
w.hasher.Reset()
// "Write implements part of the hash.Hash interface. It never returns an error."
// -- https://godoc.org/github.com/zeebo/blake3#Hasher.Write
_, _ = w.hasher.Write(input)
// Digest.Read reads data from the hasher into buffer.
// "It always fills the entire buffer and never errors."
// -- https://godoc.org/github.com/zeebo/blake3#Digest
_, _ = w.hasher.Digest().Read(buffer)
// no chance of panic, so avoid any defer cost.
w.hasherMu.Unlock()
return buffer
}
// blake3sum16 might be slower because we allocate a new hasher every time, but
// it is more conenient for writing debug code. It returns
// a 16 byte hash as a hexidecimal string.
func blake3sum16(input []byte) string {
hasher := blake3.New()
_, _ = hasher.Write(input)
var buf [16]byte
_, _ = hasher.Digest().Read(buf[0:])
return fmt.Sprintf("%x", buf)
}
// cryptoRandInt64 uses crypto/rand to get an random int64
func cryptoRandInt64() int64 {
c := 8
b := make([]byte, c)
_, err := cryptorand.Read(b)
if err != nil {
panic(err)
}
r := int64(binary.LittleEndian.Uint64(b))
return r
}

47
blake3_test.go Normal file
View file

@ -0,0 +1,47 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"testing"
"encoding/hex"
)
func TestBlake3Hasher(t *testing.T) {
hasher := NewBlake3Hasher()
hash := make([]byte, 16)
input := []byte("hello world")
hash = hasher.CryptoHash(input, hash)
expected := "d74981efa70a0c880b8d8c1985d075db"
observed := hex.EncodeToString(hash)
if observed != expected {
panic(fmt.Sprintf("expected hash:'%v' but observed hash '%v'", expected, observed))
}
obs2 := blake3sum16(input)
if obs2 != expected {
panic(fmt.Sprintf("expected hash:'%v' but observed hash from blake2sum16: '%v'", expected, obs2))
}
}
func TestCryptoRandInt64(t *testing.T) {
rnd := cryptoRandInt64()
if rnd == 0 {
panic("cryptoRandInt64() gave 0, very high odds it has broken")
}
}

421
bluegreentx.go Normal file
View file

@ -0,0 +1,421 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"github.com/pilosa/pilosa/v2/roaring"
)
// blueGreenTx runs two Tx together and notices differences in their output.
// By convention, the 'b' Tx is the output that is returned to caller.
type blueGreenTx struct {
a Tx
b Tx // b's output is returned
idx *Index
}
func newBlueGreenTx(a, b Tx, idx *Index) *blueGreenTx {
return &blueGreenTx{a: a, b: b, idx: idx}
}
var _ = newBlueGreenTx // keep linter happy
var _ Tx = (*blueGreenTx)(nil)
func (c *blueGreenTx) Readonly() bool {
a := c.a.Readonly()
b := c.b.Readonly()
if a != b {
panic(fmt.Sprintf("a=%v, but b =%v", a, b))
}
return b
}
func (c *blueGreenTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
return c.b.NewTxIterator(index, field, view, shard)
}
func (c *blueGreenTx) Pointer() string {
return fmt.Sprintf("%p", c)
}
func (c *blueGreenTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
c.a.IncrementOpN(index, field, view, shard, changedN)
c.b.IncrementOpN(index, field, view, shard, changedN)
}
func (c *blueGreenTx) Rollback() {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
c.a.Rollback()
c.b.Rollback()
}
func (c *blueGreenTx) Commit() error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
errA := c.a.Commit()
_ = errA
errB := c.b.Commit()
compareErrors(errA, errB)
return errB
}
func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
a, errA := c.a.RoaringBitmap(index, field, view, shard)
_, _ = a, errA
b, errB := c.b.RoaringBitmap(index, field, view, shard)
compareErrors(errA, errB)
return b, errB
}
func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
a, errA := c.a.Container(index, field, view, shard, key)
b, errB := c.b.Container(index, field, view, shard, key)
compareErrors(errA, errB)
err = a.BitwiseCompare(b)
panicOn(err)
return b, errB
}
func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
errA := c.a.PutContainer(index, field, view, shard, key, rc)
errB := c.b.PutContainer(index, field, view, shard, key, rc)
compareErrors(errA, errB)
/* draft idea of how to check the full databases afterwards:
hashA := c.a.RootHashString()
hashB := c.b.RootHashString()
if hashA != hashB {
panic(fmt.Sprintf("hashA = '%v' but hashB = '%v'", hashA, hashB))
}
*/
return errB
}
func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
// remember where the iterator started, so we can replay it a second time.
rit2 := rit.Clone()
changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize)
if changedA != changedB {
panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB))
}
if len(rowSetA) != len(rowSetB) {
panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB))
}
for k, va := range rowSetA {
vb, ok := rowSetB[k]
if !ok {
panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB))
}
if va != vb {
panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb))
}
}
compareErrors(errA, errB)
//compareDatabases(c.a, c.b)
return changedB, rowSetB, errB
}
/* // TODO: get a database-wide checksum working
func compareDatabases(a, b Tx) {
index, field, view, shard := "i", "f", "v", uint64(0)
ha, errA := a.WholeDatabaseBlake3Hash(index, field, view, shard)
panicOn(errA)
hb, errB := b.WholeDatabaseBlake3Hash(index, field, view, shard)
panicOn(errB)
if ha != hb {
panic(fmt.Sprintf("a.WholeDatabaseBlake3Hash(%T) = '%v' but b.WholeDatabaseBlake3Hash(%T) = '%v'", a, ha, b, hb))
}
}
*/
func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
errA := c.a.RemoveContainer(index, field, view, shard, key)
errB := c.b.RemoveContainer(index, field, view, shard, key)
compareErrors(errA, errB)
return errB
}
func (c *blueGreenTx) UseRowCache() bool {
return c.b.UseRowCache()
}
func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Add() panic '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, stack())
panic(r)
}
}()
// must copy a before calling Add(), since RoaringTx.Add() uses roaring.DirectAddN()
// which modifies the input array a.
a2 := make([]uint64, len(a))
copy(a2, a)
ach, errA := c.a.Add(index, field, view, shard, batched, a...)
_, _ = ach, errA
bch, errB := c.b.Add(index, field, view, shard, batched, a2...)
if ach != bch {
panic(fmt.Sprintf("Add() difference, ach=%v, but bch=%v; errA='%v'; errB='%v'", ach, bch, errA, errB))
}
compareErrors(errA, errB)
return bch, errB
}
func compareErrors(errA, errB error) {
switch {
case errA == nil && errB == nil:
// OK
case errA == nil:
panic(fmt.Sprintf("errA is nil, but errB = %#v", errB))
case errB == nil:
panic(fmt.Sprintf("errB is nil, but errA = %#v", errA))
default:
ae := errA.Error()
be := errB.Error()
if ae != be {
panic(fmt.Sprintf("errA is '%v', but errB is '%v'", ae, be))
}
}
}
func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
ach, errA := c.a.Remove(index, field, view, shard, a...)
_, _ = ach, errA
bch, errB := c.b.Remove(index, field, view, shard, a...)
compareErrors(errA, errB)
return bch, errB
}
func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
ax, errA := c.a.Contains(index, field, view, shard, key)
_, _ = ax, errA
bx, errB := c.b.Contains(index, field, view, shard, key)
compareErrors(errA, errB)
return bx, errB
}
func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
// TODO: need to return a blueGreenIterator too, that does close/next operations on both A and B.
ait, afound, errA := c.a.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
_, _, _ = ait, afound, errA
bit, bfound, errB := c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
compareErrors(errA, errB)
return bit, bfound, errB
}
func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
errA := c.a.ForEach(index, field, view, shard, fn)
_ = errA
errB := c.b.ForEach(index, field, view, shard, fn)
_ = errB
compareErrors(errA, errB)
return errB
}
func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
errA := c.a.ForEachRange(index, field, view, shard, start, end, fn)
_ = errA
errB := c.b.ForEachRange(index, field, view, shard, start, end, fn)
_ = errB
compareErrors(errA, errB)
return errB
}
func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
a, errA := c.a.Count(index, field, view, shard)
_, _ = a, errA
b, errB := c.b.Count(index, field, view, shard)
_, _ = b, errB
compareErrors(errA, errB)
return b, errB
}
func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
a, errA := c.a.Max(index, field, view, shard)
_, _ = a, errA
b, errB := c.b.Max(index, field, view, shard)
_, _ = b, errB
compareErrors(errA, errB)
return b, errB
}
func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
amin, afound, errA := c.a.Min(index, field, view, shard)
_, _, _ = amin, afound, errA
bmin, bfound, errB := c.b.Min(index, field, view, shard)
_, _, _ = bmin, bfound, errB
compareErrors(errA, errB)
return bmin, bfound, errB
}
func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
errA := c.a.UnionInPlace(index, field, view, shard, others...)
errB := c.b.UnionInPlace(index, field, view, shard, others...)
compareErrors(errA, errB)
return errB
}
func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
a, errA := c.a.CountRange(index, field, view, shard, start, end)
b, errB := c.b.CountRange(index, field, view, shard, start, end)
if a != b {
panic(fmt.Sprintf("a = %v, but b = %v", a, b))
}
compareErrors(errA, errB)
return b, errB
}
func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
a, errA := c.a.OffsetRange(index, field, view, shard, offset, start, end)
b, errB := c.b.OffsetRange(index, field, view, shard, offset, start, end)
err = roaringBitmapDiff(a, b)
panicOn(err)
compareErrors(errA, errB)
return b, errB
}

16
cache.go
View file

@ -350,6 +350,13 @@ type PairField struct {
Field string
}
func (p PairField) Clone() (r PairField) {
return PairField{
Pair: p.Pair,
Field: p.Field,
}
}
// ToTable implements the ToTabler interface.
func (p PairField) ToTable() (*pb.TableResponse, error) {
return pb.RowsToTable(p, 1)
@ -469,6 +476,15 @@ type PairsField struct {
Field string
}
func (p *PairsField) Clone() (r *PairsField) {
r = &PairsField{
Pairs: make([]Pair, len(p.Pairs)),
Field: p.Field,
}
copy(r.Pairs, p.Pairs)
return
}
// ToTable implements the ToTabler interface.
func (p *PairsField) ToTable() (*pb.TableResponse, error) {
return pb.RowsToTable(p, len(p.Pairs))

277
catcher.go Normal file
View file

@ -0,0 +1,277 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"github.com/pilosa/pilosa/v2/roaring"
)
// catcher is useful to report error locations with a
// stack dump before the complexity
// of the executor_test swallows up
// the location of a panic.
type catcherTx struct {
b *BadgerTx
}
func newCatcherTx(b *BadgerTx) *catcherTx {
return &catcherTx{b: b}
}
func init() {
// keep golangci-lint happy
_ = newCatcherTx
}
var _ Tx = (*catcherTx)(nil)
func (c *catcherTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
c.b.IncrementOpN(index, field, view, shard, changedN)
}
func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
return c.b.NewTxIterator(index, field, view, shard)
}
func (c *catcherTx) WholeDatabaseBlake3Hash(index, field, view string, shard uint64) (hash string, err error) {
return c.b.WholeDatabaseBlake3Hash(index, field, view, shard)
}
func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
}
func (c *catcherTx) Readonly() bool {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Readonly() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Readonly()
}
func (tx *catcherTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
func (c *catcherTx) Rollback() {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
c.b.Rollback()
}
func (c *catcherTx) Commit() error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Commit()
}
func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.RoaringBitmap(index, field, view, shard)
}
func (c *catcherTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Container(index, field, view, shard, key)
}
func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.PutContainer(index, field, view, shard, key, rc)
}
func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.RemoveContainer(index, field, view, shard, key)
}
func (c *catcherTx) UseRowCache() bool {
return c.b.UseRowCache()
}
func (c *catcherTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Add() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Add(index, field, view, shard, batched, a...)
}
func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Remove(index, field, view, shard, a...)
}
func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Contains(index, field, view, shard, key)
}
func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
}
func (c *catcherTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.ForEach(index, field, view, shard, fn)
}
func (c *catcherTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.ForEachRange(index, field, view, shard, start, end, fn)
}
func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Count(index, field, view, shard)
}
func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Max(index, field, view, shard)
}
func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.Min(index, field, view, shard)
}
func (c *catcherTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.UnionInPlace(index, field, view, shard, others...)
}
func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.CountRange(index, field, view, shard, start, end)
}
func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
panic(r)
}
}()
return c.b.OffsetRange(index, field, view, shard, offset, start, end)
}

18
cluster_internal_test.go
View file

@ -161,7 +161,7 @@ func TestFragSources(t *testing.T) {
// Obtain transaction.
tx := &RoaringTx{Index: idx}
defer func() { _ = tx.Rollback() }()
defer tx.Rollback()
field, err := idx.CreateFieldIfNotExists("f", OptFieldTypeDefault())
if err != nil {
@ -787,6 +787,7 @@ func TestCluster_ResizeStates(t *testing.T) {
if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil {
t.Fatalf("creating field: %v", err)
}
// Each tc.SetBit starts and commits its own Tx.
if err := tc.SetBit("i", "f", 1, 101, nil); err != nil {
t.Fatalf("setting bit: %v", err)
}
@ -804,6 +805,12 @@ func TestCluster_ResizeStates(t *testing.T) {
t.Fatal(err)
}
idx0 := node0.holder.Index("i")
if idx0 == nil {
t.Fatal(`idx0 was nil, could not retrieve Index("i")`)
}
//idx0.Dump("node0")
// addNode needs to block until the resize process has completed.
if err := tc.addNode(); err != nil {
t.Fatalf("adding node: %v", err)
@ -816,6 +823,7 @@ func TestCluster_ResizeStates(t *testing.T) {
} else if node1.State() != ClusterStateNormal {
t.Errorf("expected node1 state: %v, but got: %v", ClusterStateNormal, node1.State())
}
// INVAR: after node1.State() is normal, the rebalancing should have been done.
expectedTop := &Topology{
nodeIDs: []string{node0.Node.ID, node1.Node.ID},
@ -834,11 +842,19 @@ func TestCluster_ResizeStates(t *testing.T) {
node1View := node1Field.view("standard")
node1Fragment := node1View.Fragment(1)
idx1 := node1.holder.Index("i")
if idx1 == nil {
t.Fatal(`idx1 was nil, could not retrieve Index("i")`)
}
//idx0.Dump("after rebalance, node0")
//idx1.Dump("after rebalance, node1")
// Ensure checksums are the same.
if chksum, err := node1Fragment.Checksum(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
// badger red: TestCluster_ResizeStates/Multiple_nodes,_with_data: cluster_internal_test.go:841: expected standard view checksum to match: ef46db3751d8e999 - fad4de25ee696ca0
}
// Close TestCluster.

3
ctl/server.go
View file

@ -85,4 +85,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
// Profiling
flags.IntVar(&srv.Config.Profile.BlockRate, "profile.block-rate", srv.Config.Profile.BlockRate, "Sampling rate for goroutine blocking profiler. One sample per <rate> ns.")
flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/<rate> of events.")
// Transactional storage engine
flags.StringVarP(&srv.Config.Txsrc, "tx", "", "roaring", "transaction/storage to use: one of roaring, rbf, badger, rbf_roaring, roaring_rbf, badger_roaring, roaring_badger, badger_rbf, or rbf_badger")
}

143
executor.go
View file

@ -180,8 +180,14 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
return resp, ErrIndexNotFound
}
needWriteTxn := false
nw := q.WriteCallN()
if nw > 0 {
needWriteTxn = true
}
// Verify that the number of writes do not exceed the maximum.
if e.MaxWritesPerRequest > 0 && q.WriteCallN() > e.MaxWritesPerRequest {
if e.MaxWritesPerRequest > 0 && nw > e.MaxWritesPerRequest {
return resp, ErrTooManyWrites
}
@ -210,12 +216,11 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
}
}
// TODO: Determine if query is read-only.
tx, err := e.Holder.Begin(true)
tx, err := e.Holder.BeginTx(needWriteTxn, idx)
if err != nil {
return resp, err
}
defer func() { _ = tx.Rollback() }()
defer tx.Rollback()
results, err := e.execute(ctx, tx, index, q, shards, opt)
if err != nil {
@ -267,19 +272,75 @@ func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shar
// Translate response objects from ids to keys, if necessary.
// No need to translate a remote call.
if !opt.Remote {
// only translateResults if this local node is the final destination. only string/column keys.
if err := e.translateResults(ctx, index, idx, q.Calls, results); err != nil {
return resp, err
} else if err := validateQueryContext(ctx); err != nil {
return resp, err
}
}
// Must copy out of Tx data before Commiting, because it will become invalid afterwards.
respSafeNoTxData := e.safeCopy(resp)
// Commit transaction.
if err := tx.Commit(); err != nil {
return resp, err
return respSafeNoTxData, err
}
return respSafeNoTxData, nil
}
return resp, nil
// safeCopy copies everything in resp that has Bitmap material,
// to avoid anything coming from the mmap-ed Tx storage.
func (e *executor) safeCopy(resp QueryResponse) (out QueryResponse) {
out = QueryResponse{
// not transactional, from attribute storage so no need to clone these:
ColumnAttrSets: resp.ColumnAttrSets, // []*ColumnAttrSet
Err: resp.Err, // error
Profile: resp.Profile, // *tracing.Profile
}
// Results can contain *roaring.Bitmap, so need to copy from Tx mmap-ed memory.
for _, v := range resp.Results {
switch x := v.(type) {
case *Row:
rowSafe := x.Clone()
out.Results = append(out.Results, rowSafe)
case bool:
out.Results = append(out.Results, x)
case nil:
out.Results = append(out.Results, nil)
case uint64:
out.Results = append(out.Results, x) // for counts
case *PairsField:
// no bitmap material, so should be ok to skip Clone()
out.Results = append(out.Results, x)
case PairField: // not PairsField but PairField
// no bitmap material, so should be ok to skip Clone()
out.Results = append(out.Results, x)
case ValCount:
// no bitmap material, so should be ok to skip Clone()
out.Results = append(out.Results, x)
case SignedRow:
// has *Row in it, so has Bitmap material, and very likely needs Clone.
y := x.Clone()
out.Results = append(out.Results, *y)
case GroupCount:
// no bitmap material, so should be ok to skip Clone()
out.Results = append(out.Results, x)
case []GroupCount:
out.Results = append(out.Results, x)
case RowIdentifiers:
// no bitmap material, so should be ok to skip Clone()
out.Results = append(out.Results, x)
case RowIDs:
// defined as: type RowIDs []uint64
// so does not contain bitmap material, and
// should not need to be cloned.
out.Results = append(out.Results, x)
default:
panic(fmt.Sprintf("handle %T here", v))
}
}
return
}
// readColumnAttrSets returns a list of column attribute objects by id.
@ -308,6 +369,7 @@ func (e *executor) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttr
// handlePreCalls traverses the call tree looking for calls that need
// precomputed values. Right now, that's just Distinct.
func (e *executor) handlePreCalls(ctx context.Context, tx Tx, index string, c *pql.Call, shards []uint64, opt *execOptions) error {
if c.Name == "Precomputed" {
idx := c.Args["valueidx"].(int64)
if idx >= 0 && idx < int64(len(opt.EmbeddedData)) {
@ -457,6 +519,7 @@ func (e *executor) execute(ctx context.Context, tx Tx, index string, q *pql.Quer
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
for i, call := range q.Calls {
if err := validateQueryContext(ctx); err != nil {
return nil, err
}
@ -1019,6 +1082,7 @@ func (e *executor) executeAllCallMapReduce(ctx context.Context, tx Tx, index str
// executeIncludesColumnCallShard
func (e *executor) executeIncludesColumnCallShard(ctx context.Context, tx Tx, index string, c *pql.Call, shard uint64, column uint64) (bool, error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeIncludesColumnCallShard")
defer span.Finish()
@ -1308,7 +1372,8 @@ func (e *executor) executeBitmapCall(ctx context.Context, tx Tx, index string, c
reduceFn := func(ctx context.Context, prev, v interface{}) interface{} {
other, _ := prev.(*Row)
if other == nil {
other = NewRow()
other = NewRow() // bug! this row ends up containing Badger Txn data that should be accessed outside the Txn.
}
if err := ctx.Err(); err != nil {
return err
@ -1823,6 +1888,21 @@ type RowIdentifiers struct {
field string
}
func (r *RowIdentifiers) Clone() (clone *RowIdentifiers) {
clone = &RowIdentifiers{
field: r.field,
}
if r.Rows != nil {
clone.Rows = make([]uint64, len(r.Rows))
copy(clone.Rows, r.Rows)
}
if r.Keys != nil {
clone.Keys = make([]string, len(r.Keys))
copy(clone.Keys, r.Keys)
}
return
}
// ToTable implements the ToTabler interface.
func (r RowIdentifiers) ToTable() (*pb.TableResponse, error) {
var n int
@ -2041,6 +2121,20 @@ type FieldRow struct {
Value *int64 `json:"value,omitempty"`
}
func (fr *FieldRow) Clone() (clone *FieldRow) {
clone = &FieldRow{
Field: fr.Field,
RowID: fr.RowID,
RowKey: fr.RowKey,
}
if fr.Value != nil {
// deep copy, for safety.
v := *fr.Value
clone.Value = &v
}
return
}
// MarshalJSON marshals FieldRow to JSON such that
// either a Key or an ID is included.
func (fr FieldRow) MarshalJSON() ([]byte, error) {
@ -2138,6 +2232,18 @@ type GroupCount struct {
Sum int64 `json:"sum"`
}
func (g *GroupCount) Clone() (r *GroupCount) {
r = &GroupCount{
Group: make([]FieldRow, len(g.Group)),
Count: g.Count,
Sum: g.Sum,
}
for i := range g.Group {
r.Group[i] = *(g.Group[i].Clone())
}
return
}
// mergeGroupCounts merges two slices of GroupCounts throwing away any that go
// beyond the limit. It assume that the two slices are sorted by the row ids in
// the fields of the group counts. It may modify its arguments.
@ -2642,6 +2748,7 @@ func (e *executor) executeRowShard(ctx context.Context, tx Tx, index string, c *
// Simply return row if times are not set.
if c.Name == "Row" && timeNotSet {
frag := e.Holder.fragment(index, fieldName, viewStandard, shard)
if frag == nil {
return NewRow(), nil
@ -3771,6 +3878,7 @@ func (e *executor) executeSetColumnAttrs(ctx context.Context, tx Tx, index strin
delete(attrs, "field")
// Set attributes.
if err := idx.ColumnAttrStore().SetAttrs(col, attrs); err != nil {
return err
}
@ -4654,6 +4762,15 @@ type SignedRow struct {
field string
}
func (s *SignedRow) Clone() (r *SignedRow) {
r = &SignedRow{
Neg: s.Neg.Clone(), // Row.Clone() returns nil for nil.
Pos: s.Pos.Clone(),
field: s.field,
}
return
}
// Field returns the field name associated to the signed row.
func (s *SignedRow) Field() string {
return s.field
@ -4768,6 +4885,18 @@ type ValCount struct {
Count int64 `json:"count"`
}
func (v *ValCount) Clone() (r *ValCount) {
r = &ValCount{
Val: v.Val,
FloatVal: v.FloatVal,
Count: v.Count,
}
if v.DecimalVal != nil {
r.DecimalVal = v.DecimalVal.Clone()
}
return
}
// ToTable implements the ToTabler interface.
func (v ValCount) ToTable() (*pb.TableResponse, error) {
return pb.RowsToTable(&v, 1)

12
executor_internal_test.go
View file

@ -137,12 +137,6 @@ func TestExecutor_TranslateRowsOnBool(t *testing.T) {
holder := NewHolder(DefaultPartitionN)
defer holder.Close()
tx, err := holder.Begin(true)
if err != nil {
t.Fatal(err)
}
defer func() { _ = tx.Rollback() }()
e := &executor{
Holder: holder,
Cluster: NewTestCluster(1),
@ -157,6 +151,12 @@ func TestExecutor_TranslateRowsOnBool(t *testing.T) {
t.Fatalf("creating index: %v", err)
}
tx, err := holder.BeginTx(writable, idx)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
fb, errb := idx.CreateField("b", OptFieldTypeBool())
_, errbk := idx.CreateField("bk", OptFieldTypeBool(), OptFieldKeys())
if errb != nil || errbk != nil {

14
executor_test.go
View file

@ -41,6 +41,9 @@ import (
"github.com/pkg/errors"
)
// writable initializes Tx that update, use !writable for read-only.
const writable = true
var (
TempDir = getTempDirString()
)
@ -142,6 +145,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c[0].Server.Holder()}
hldr.SetBit("i", "general", 10, 1)
hldr.SetBit("i", "general", 10, 2)
hldr.SetBit("i", "general", 10, 3)
@ -223,7 +227,6 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
hldr.SetBit("i", "general", 10, 1)
hldr.SetBit("i", "general", 10, ShardWidth+1)
hldr.SetBit("i", "general", 10, ShardWidth+2)
hldr.SetBit("i", "general", 11, 1)
hldr.SetBit("i", "general", 11, 2)
hldr.SetBit("i", "general", 11, ShardWidth+2)
@ -904,11 +907,11 @@ func TestExecutor_Execute_SetValue(t *testing.T) {
}
// Obtain transaction.
tx, err := hldr.Begin(false)
tx, err := hldr.BeginTx(!writable, index.Index)
if err != nil {
t.Fatal(err)
}
defer func() { _ = tx.Rollback() }()
defer tx.Rollback()
f := hldr.Field("i", "f")
if value, exists, err := f.Value(tx, 10); err != nil {
@ -3435,7 +3438,6 @@ func TestExecutor_Execute_Existence(t *testing.T) {
} else if bits := res.Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{ShardWidth + 2}) {
t.Fatalf("unexpected columns after Not: %+v", bits)
}
// Reopen cluster to ensure existence field is reloaded.
if err := c[0].Reopen(); err != nil {
t.Fatal(err)
@ -3820,7 +3822,6 @@ func TestExecutor_Execute_ClearRow(t *testing.T) {
if res := responses[1].Results[0].(bool); !res {
t.Fatalf("unexpected clear row result: %+v", res)
}
// Clear the row again and ensure we get a `false` response.
if res := responses[2].Results[0].(bool); res {
t.Fatalf("unexpected clear row result: %+v", res)
@ -4023,6 +4024,7 @@ func TestExecutor_Execute_ClearRow(t *testing.T) {
// Ensure a row can be set.
func TestExecutor_Execute_SetRow(t *testing.T) {
t.Run("Set_NewRow", func(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
@ -4824,6 +4826,7 @@ func TestExecutor_ForeignIndex(t *testing.T) {
}
join := c.Query(t, "parent", `Intersect(Row(general=3), Distinct(Row(color="blue"), index="child", field="parent_id"))`).Results[0].(*pilosa.Row)
if !reflect.DeepEqual(join.Keys, []string{"one"}) {
t.Fatalf("unexpected keys: %v", join.Keys)
}
@ -4870,6 +4873,7 @@ func TestExecutor_Execute_GroupBy(t *testing.T) {
{12, 2},
{12, ShardWidth + 2},
})
c.ImportBits(t, "i", "sub", [][2]uint64{
{100, 0},
{100, 1},

4
field.go
View file

@ -1737,7 +1737,7 @@ func (f *Field) importValue(tx Tx, columnIDs []uint64, values []int64, options *
return nil
}
func (f *Field) importRoaring(ctx context.Context, data []byte, shard uint64, viewName string, clear bool) error {
func (f *Field) importRoaring(ctx context.Context, tx Tx, data []byte, shard uint64, viewName string, clear bool) error {
span, ctx := tracing.StartSpanFromContext(ctx, "Field.importRoaring")
defer span.Finish()
@ -1754,7 +1754,7 @@ func (f *Field) importRoaring(ctx context.Context, data []byte, shard uint64, vi
if err != nil {
return errors.Wrap(err, "creating fragment")
}
if err := frag.importRoaring(ctx, data, clear); err != nil {
if err := frag.importRoaring(ctx, tx, data, clear); err != nil {
return err
}

179
fragment.go
View file

@ -129,7 +129,14 @@ type fragment struct {
// Cache for row counts.
CacheType string // passed in by field
cache cache
// cache keeps a local rowid,count ranking:
// telling us which is the most populated rows in that field.
// Is only on "set fields" with rowCache enabled. So
// BSI, mutex, bool fields do not have this.
// Good: it Only has a string and a count, so cannot use Tx memory.
cache cache
CacheSize uint32
// Stats reporting.
@ -160,6 +167,15 @@ type fragment struct {
stats stats.StatsClient
bitmapInfo *roaring.BitmapInfo
// txTestingOnly: this looks gross.
// Nonetheless, it allowed us to
// integrate Tx into the
// fragment_internal_test.go suite
// and not break the world all at once.
//
// Only for testing, obviously.
txTestingOnly Tx
}
// newFragment returns a new instance of Fragment.
@ -192,6 +208,10 @@ type FragmentInfo struct {
BlockChecksums []FragmentBlock `json:"BlockChecksums,omitempty"`
}
func (f *fragment) Index() *Index {
return f.holder.Index(f.index)
}
func (f *fragment) inspect(params InspectRequestParams) (fi FragmentInfo) {
if f.bitmapInfo == nil {
fi.BitmapInfo = f.storage.Info(params.Containers)
@ -529,16 +549,20 @@ func (f *fragment) mustRow(tx Tx, rowID uint64) *Row {
// unprotectedRow returns a row from the row cache if available or from storage
// (updating the cache).
func (f *fragment) unprotectedRow(tx Tx, rowID uint64) (*Row, error) {
r, ok := f.rowCache.Fetch(rowID)
if ok && r != nil {
return r, nil
useRowCache := tx.UseRowCache()
if useRowCache {
r, ok := f.rowCache.Fetch(rowID)
if ok && r != nil {
return r, nil
}
}
row, err := f.rowFromStorage(tx, rowID)
if err != nil {
return nil, err
}
f.rowCache.Add(rowID, row)
if useRowCache {
f.rowCache.Add(rowID, row)
}
return row, nil
}
@ -559,7 +583,7 @@ func (f *fragment) rowFromStorage(tx Tx, rowID uint64) (*Row, error) {
row := &Row{
segments: []rowSegment{{
data: data,
data: data, // this data contains BadgerTx data, which should not survive Txn commit.
shard: f.shard,
writable: true,
}},
@ -602,7 +626,7 @@ func (f *fragment) handleMutex(tx Tx, rowID, columnID uint64) error {
// unprotectedSetBit TODO should be replaced by an invocation of importPositions with a single bit to set.
func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed bool, err error) {
changed = false
// Determine the position of the bit in the storage.
pos, err := f.pos(rowID, columnID)
if err != nil {
@ -610,7 +634,10 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo
}
// Write to storage.
if changed, err = tx.Add(f.index, f.field, f.view, f.shard, pos); err != nil {
changeCount := 0
changeCount, err = tx.Add(f.index, f.field, f.view, f.shard, doBatched, pos)
changed = changeCount > 0
if err != nil {
return false, errors.Wrap(err, "writing")
}
@ -623,7 +650,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo
delete(f.checksums, int(rowID/HashBlockSize))
// Increment number of operations until snapshot is required.
f.incrementOpN(1)
tx.IncrementOpN(f.index, f.field, f.view, f.shard, 1)
// If we're using a cache, update it. Otherwise skip the
// possibly-expensive count operation.
@ -671,20 +698,23 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b
}
// Write to storage.
if changed, err = tx.Remove(f.index, f.field, f.view, f.shard, pos); err != nil {
changeCount := 0
if changeCount, err = tx.Remove(f.index, f.field, f.view, f.shard, pos); err != nil {
return false, errors.Wrap(err, "writing")
}
// Don't update the cache if nothing changed.
if !changed {
return changed, nil
if changeCount <= 0 {
return false, nil
} else {
changed = true
}
// Invalidate block checksum.
delete(f.checksums, int(rowID/HashBlockSize))
// Increment number of operations until snapshot is required.
f.incrementOpN(1)
tx.IncrementOpN(f.index, f.field, f.view, f.shard, 1)
// If we're using a cache, update it. Otherwise skip the
// possibly-expensive count operation.
@ -997,15 +1027,17 @@ func (f *fragment) importSetValue(tx Tx, columnID uint64, bitDepth uint, value i
}
if uvalue&(1<<i) != 0 {
if c, err := tx.Add(f.index, f.field, f.view, f.shard, bit); err != nil {
c, err := tx.Add(f.index, f.field, f.view, f.shard, !doBatched, bit)
if err != nil {
return changed, errors.Wrap(err, "adding")
} else if c {
} else if c > 0 {
changed++
}
} else {
if c, err := tx.Remove(f.index, f.field, f.view, f.shard, bit); err != nil {
changeCount, err := tx.Remove(f.index, f.field, f.view, f.shard, bit)
if err != nil {
return changed, errors.Wrap(err, "removing")
} else if c {
} else if changeCount > 0 {
changed++
}
}
@ -1017,13 +1049,13 @@ func (f *fragment) importSetValue(tx Tx, columnID uint64, bitDepth uint, value i
} else if clear {
if c, err := tx.Remove(f.index, f.field, f.view, f.shard, p); err != nil {
return changed, errors.Wrap(err, "removing not-null from storage")
} else if c {
} else if c > 0 {
changed++
}
} else {
if c, err := tx.Add(f.index, f.field, f.view, f.shard, p); err != nil {
if c, err := tx.Add(f.index, f.field, f.view, f.shard, !doBatched, p); err != nil {
return changed, errors.Wrap(err, "adding not-null to storage")
} else if c {
} else if c > 0 {
changed++
}
}
@ -1034,13 +1066,13 @@ func (f *fragment) importSetValue(tx Tx, columnID uint64, bitDepth uint, value i
} else if value >= 0 || clear {
if c, err := tx.Remove(f.index, f.field, f.view, f.shard, p); err != nil {
return changed, errors.Wrap(err, "removing sign from storage")
} else if c {
} else if c > 0 {
changed++
}
} else {
if c, err := tx.Add(f.index, f.field, f.view, f.shard, p); err != nil {
if c, err := tx.Add(f.index, f.field, f.view, f.shard, !doBatched, p); err != nil {
return changed, errors.Wrap(err, "adding sign to storage")
} else if c {
} else if c > 0 {
changed++
}
}
@ -1889,8 +1921,17 @@ func (f *fragment) Blocks() ([]FragmentBlock, error) {
var a []FragmentBlock
// Initialize the iterator.
itr := f.storage.Iterator()
idx := f.holder.Index(f.index)
if idx == nil {
panic(fmt.Sprintf("index was nil in fragment.Blocks(): f.index='%v'; f.holder.indexes='%#v'\n", f.index, f.holder.indexes))
}
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f})
defer tx.Rollback()
// no Commit below, b/c is read-only.
itr := tx.NewTxIterator(f.index, f.field, f.view, f.shard)
defer itr.Close()
itr.Seek(0)
// Initialize block hasher.
@ -1967,7 +2008,13 @@ func (f *fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i
func (f *fragment) blockData(id int) (rowIDs, columnIDs []uint64, err error) {
f.mu.Lock()
defer f.mu.Unlock()
if err := f.storage.ForEachRange(uint64(id)*HashBlockSize*ShardWidth, (uint64(id)+1)*HashBlockSize*ShardWidth, func(i uint64) error {
idx := f.holder.Index(f.index)
tx := idx.Txf.NewTx(Txo{Write: !writable, Index: idx})
defer tx.Rollback()
// readonly, so no Commit()
if err := tx.ForEachRange(f.index, f.field, f.view, f.shard, uint64(id)*HashBlockSize*ShardWidth, (uint64(id)+1)*HashBlockSize*ShardWidth, func(i uint64) error {
rowIDs = append(rowIDs, i/ShardWidth)
columnIDs = append(columnIDs, i%ShardWidth)
return nil
@ -2101,7 +2148,7 @@ func (f *fragment) mergeBlock(tx Tx, id int, data []pairSet) (sets, clears []pai
rowSet[clears[0].rowIDs[i]] = struct{}{}
clears[0].columnIDs[i] += clears[0].rowIDs[i] * ShardWidth
}
err = f.importPositions(sets[0].columnIDs, clears[0].columnIDs, rowSet)
err = f.importPositions(tx, sets[0].columnIDs, clears[0].columnIDs, rowSet)
return sets[1:], clears[1:], err
}
@ -2149,9 +2196,9 @@ func (f *fragment) bulkImportStandard(tx Tx, rowIDs, columnIDs []uint64, options
f.mu.Lock()
defer f.mu.Unlock()
if options.Clear {
err = f.importPositions(nil, positions, rowSet)
err = f.importPositions(tx, nil, positions, rowSet)
} else {
err = f.importPositions(positions, nil, rowSet)
err = f.importPositions(tx, positions, nil, rowSet)
}
return errors.Wrap(err, "bulkImportStandard")
}
@ -2164,26 +2211,31 @@ func (f *fragment) bulkImportStandard(tx Tx, rowIDs, columnIDs []uint64, options
// importPositions tries to intelligently decide whether or not to do a full
// snapshot of the fragment or just do in-memory updates while appending
// operations to the op log.
func (f *fragment) importPositions(set, clear []uint64, rowSet map[uint64]struct{}) error {
func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64]struct{}) error {
//tx.AddN()
err := f.gen.Transaction(&f.storage.OpWriter, func() error {
if len(set) > 0 {
f.stats.Count(MetricImportingN, int64(len(set)), 1)
changedN, err := f.storage.AddN(set...) // TODO benchmark Add/RemoveN behavior with sorted/unsorted positions
// TODO benchmark Add/RemoveN behavior with sorted/unsorted positions
// Note: AddN() avoids writing to the op-log. While Add() does.
changedN, err := tx.Add(f.index, f.field, f.view, f.shard, !doBatched, set...)
if err != nil {
return errors.Wrap(err, "adding positions")
}
f.stats.Count(MetricImportedN, int64(changedN), 1)
f.incrementOpN(changedN)
tx.IncrementOpN(f.index, f.field, f.view, f.shard, changedN)
}
if len(clear) > 0 {
f.stats.Count(MetricClearingN, int64(len(clear)), 1)
changedN, err := f.storage.RemoveN(clear...)
changedN, err := tx.Remove(f.index, f.field, f.view, f.shard, clear...)
if err != nil {
return errors.Wrap(err, "clearing positions")
}
f.stats.Count(MetricClearedN, int64(changedN), 1)
f.incrementOpN(changedN)
tx.IncrementOpN(f.index, f.field, f.view, f.shard, changedN)
}
// Update cache counts for all affected rows.
@ -2192,7 +2244,14 @@ func (f *fragment) importPositions(set, clear []uint64, rowSet map[uint64]struct
delete(f.checksums, int(rowID/HashBlockSize))
if f.CacheType != CacheTypeNone {
n := f.storage.CountRange(rowID*ShardWidth, (rowID+1)*ShardWidth)
start := rowID * ShardWidth
end := (rowID + 1) * ShardWidth
n, err := tx.CountRange(f.index, f.field, f.view, f.shard, start, end)
if err != nil {
return errors.Wrap(err, "CountRange")
}
f.cache.BulkAdd(rowID, n)
}
@ -2275,10 +2334,10 @@ func (f *fragment) bulkImportMutex(tx Tx, rowIDs, columnIDs []uint64) error {
toSet := rowIDs[:i]
toClear := columnIDs[:clearIdx]
return errors.Wrap(f.importPositions(toSet, toClear, rowSet), "importing positions")
return errors.Wrap(f.importPositions(tx, toSet, toClear, rowSet), "importing positions")
}
func (f *fragment) importValueSmallWrite(columnIDs []uint64, values []int64, bitDepth uint, clear bool) error {
func (f *fragment) importValueSmallWrite(tx Tx, columnIDs []uint64, values []int64, bitDepth uint, clear bool) error {
// TODO figure out how to avoid re-allocating these each time. Probably
// possible to store them on the fragment with a capacity based on
// MaxOpN. For now, we know that the total number of bits to be
@ -2310,7 +2369,7 @@ func (f *fragment) importValueSmallWrite(columnIDs []uint64, values []int64, bit
for i := uint(0); i < bitDepth+1; i++ {
rowSet[uint64(i)] = struct{}{}
}
err := f.importPositions(toSet, toClear, rowSet)
err := f.importPositions(tx, toSet, toClear, rowSet)
if err != nil {
return errors.Wrap(err, "importing positions")
}
@ -2332,7 +2391,7 @@ func (f *fragment) importValue(tx Tx, columnIDs []uint64, values []int64, bitDep
}
if len(columnIDs)*int(bitDepth+1)+f.opN < f.MaxOpN {
return errors.Wrap(f.importValueSmallWrite(columnIDs, values, bitDepth, clear), "import small write")
return errors.Wrap(f.importValueSmallWrite(tx, columnIDs, values, bitDepth, clear), "import small write")
}
// Process every value.
@ -2374,7 +2433,7 @@ func (f *fragment) importValue(tx Tx, columnIDs []uint64, values []int64, bitDep
// importRoaring imports from the official roaring data format defined at
// https://github.com/RoaringBitmap/RoaringFormatSpec or from pilosa's version
// of the roaring format. The cache is updated to reflect the new data.
func (f *fragment) importRoaring(ctx context.Context, data []byte, clear bool) error {
func (f *fragment) importRoaring(ctx context.Context, tx Tx, data []byte, clear bool) error {
span, ctx := tracing.StartSpanFromContext(ctx, "fragment.importRoaring")
defer span.Finish()
span, ctx = tracing.StartSpanFromContext(ctx, "importRoaring.AcquireFragmentLock")
@ -2382,16 +2441,22 @@ func (f *fragment) importRoaring(ctx context.Context, data []byte, clear bool) e
defer f.mu.Unlock()
span.Finish()
return f.unprotectedImportRoaring(ctx, data, clear)
return f.unprotectedImportRoaring(ctx, tx, data, clear)
}
func (f *fragment) unprotectedImportRoaring(ctx context.Context, data []byte, clear bool) error {
func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []byte, clear bool) error {
rowSize := uint64(1 << shardVsContainerExponent)
span, ctx := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits")
var changed int
var rowSet map[uint64]int
err := f.gen.Transaction(&f.storage.OpWriter, func() (err error) {
changed, rowSet, err = f.storage.ImportRoaringBits(data, clear, true, rowSize)
var rit roaring.RoaringIterator
rit, err = roaring.NewRoaringIterator(data)
if err != nil {
return err
}
changed, rowSet, err = tx.ImportRoaringBits(f.index, f.field, f.view, f.shard, rit, clear, true, rowSize)
return err
})
@ -2428,7 +2493,9 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, data []byte, cl
}
span, _ = tracing.StartSpanFromContext(ctx, "importRoaring.incrementOpN")
f.incrementOpN(changed)
tx.IncrementOpN(f.index, f.field, f.view, f.shard, changed)
span.Finish()
return nil
}
@ -2444,7 +2511,7 @@ func (f *fragment) importRoaringOverwrite(ctx context.Context, tx Tx, data []byt
}
// Union the new block data with the fragment data.
return f.unprotectedImportRoaring(ctx, data, false)
return f.unprotectedImportRoaring(ctx, tx, data, false)
}
// incrementOpN increase the operation count by one.
@ -2486,7 +2553,6 @@ func (f *fragment) snapshot() (err error) {
defer func() {
debug.SetPanicOnFault(wouldPanic)
if r := recover(); r != nil {
fmt.Printf("snapshot panic!\n")
if e2, ok := r.(error); ok {
err = e2
// special case: if we caught a page fault, we diagnose that directly. sadly,
@ -2869,6 +2935,7 @@ func (f *fragment) unprotectedRows(ctx context.Context, tx Tx, start uint64, fil
if err != nil {
return nil, err
}
defer i.Close() // must close iterators allocated on a Tx
rows := make([]uint64, 0)
var lastRow uint64 = math.MaxUint64
@ -3074,6 +3141,8 @@ func (f *fragment) foreachRow(tx Tx, filters []rowFilter, fn func(rid uint64) er
if err != nil {
return err
}
defer i.Close() // must close tx allocated iterators when done.
// Loop over the existing containers.
for i.Next() {
key, c := i.Value()
@ -3263,7 +3332,7 @@ func (s *fragmentSyncer) syncFragment() error {
for _, node := range nodes {
// Read local blocks.
if node.ID == s.Node.ID {
b, err := s.Fragment.Blocks()
b, err := s.Fragment.Blocks() // comes from Tx store, creates its own Tx.
if err != nil {
return err
}
@ -3404,7 +3473,6 @@ func (s *fragmentSyncer) syncBlock(id int) error {
defer span.Finish()
f := s.Fragment
tx := &RoaringTx{fragment: f}
// Read pairs from each remote block.
var uris []*URI
@ -3423,6 +3491,7 @@ func (s *fragmentSyncer) syncBlock(id int) error {
uris = append(uris, uri)
// Only sync the standard block.
// Does a remote fetch
rowIDs, columnIDs, err := s.Cluster.InternalClient.BlockData(ctx, &node.URI, f.index, f.field, f.view, f.shard, id)
if err != nil {
return errors.Wrap(err, "getting block")
@ -3439,12 +3508,24 @@ func (s *fragmentSyncer) syncBlock(id int) error {
return nil
}
idx := f.holder.Index(f.index)
tx := idx.Txf.NewTx(Txo{Write: writable, Index: idx})
defer tx.Rollback()
// Merge blocks together.
sets, clears, err := f.mergeBlock(tx, id, pairSets)
if err != nil {
return errors.Wrap(err, "merging")
}
// no safeCopy needed here. We are not leaking data outside the tx, because
// sets and clears only contain columnIDs.
err = tx.Commit()
if err != nil {
return err
}
// Write updates to remote blocks.
for i := 0; i < len(uris); i++ {
set, clear := sets[i], clears[i]

755
fragment_internal_test.go
View file

File diff suppressed because it is too large Load diff

13
go.mod
View file

@ -9,9 +9,12 @@ require (
github.com/benbjohnson/immutable v0.2.0
github.com/boltdb/bolt v1.3.1
github.com/cespare/xxhash v1.1.0
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
github.com/davecgh/go-spew v1.1.1
github.com/dchest/blake2b v1.0.0 // indirect
github.com/dgraph-io/badger v1.6.1-0.20191025180844-32a2548a9d85 // indirect
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/gogo/protobuf v1.2.0
github.com/golang/protobuf v1.3.3
@ -31,17 +34,17 @@ require (
github.com/satori/go.uuid v1.2.0
github.com/shirou/gopsutil v2.18.12+incompatible
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
github.com/spf13/cobra v0.0.3
github.com/spf13/cobra v0.0.5
github.com/spf13/pflag v1.0.3
github.com/spf13/viper v1.3.1
github.com/spf13/viper v1.3.2
github.com/uber-go/atomic v1.4.0 // indirect
github.com/uber/jaeger-client-go v2.16.0+incompatible
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
github.com/willoch/tago v0.0.0-20180311150625-8f2f8e8900dc // indirect
github.com/zeebo/blake3 v0.0.4
go.uber.org/atomic v1.4.0 // indirect
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 // indirect
golang.org/x/sync v0.0.0-20190423024810-112230192c58
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 // indirect
golang.org/x/text v0.3.2 // indirect
google.golang.org/grpc v1.28.0
modernc.org/mathutil v1.0.0

55
go.sum
View file

@ -5,6 +5,8 @@ github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d h1:n0G4ckjMEj7bWu
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d/go.mod h1:Rn2zM2MnHze07LwkneP48TWt6UiZhzQTwCvw6djVGfE=
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ=
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
@ -24,6 +26,7 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@ -33,9 +36,29 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dchest/blake2b v1.0.0 h1:KK9LimVmE0MjRl9095XJmKqZ+iLxWATvlcpVFRtaw6s=
github.com/dchest/blake2b v1.0.0/go.mod h1:U034kXgbJpCle2wSk5ybGIVhOSHCVLMDqOzcPEA0F7s=
github.com/dgraph-io/badger v1.6.1-0.20191025180844-32a2548a9d85 h1:oEqDRoxpep5ZlTxrAFc2yg+f0uBdUtkZE0uWsOru5bc=
github.com/dgraph-io/badger v1.6.1-0.20191025180844-32a2548a9d85/go.mod h1:cEjdIw+iaGXuQdsDymXPRcpp8yHXZ6PmwmDJajnVyJc=
github.com/dgraph-io/badger v1.6.1 h1:w9pSFNSdq/JPM1N12Fz/F/bzo993Is1W+Q7HjPzi7yg=
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361 h1:JBNM90aGLCiF9iJYvpvayMpYeW498v5ZDZqE2chqZ2A=
github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200709123515-8e896a7af361/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE=
github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl6XJI=
github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM=
github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs=
github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU=
github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA=
github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
@ -61,6 +84,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
@ -90,12 +115,16 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
@ -139,6 +168,7 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook=
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
@ -150,21 +180,30 @@ github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38=
github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o=
github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/uber/jaeger-client-go v2.16.0+incompatible h1:Q2Pp6v3QYiocMxomCaJuwQGFt7E53bPYqEgug/AoBtY=
@ -172,7 +211,16 @@ github.com/uber/jaeger-client-go v2.16.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW
github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/willoch/tago v0.0.0-20180311150625-8f2f8e8900dc h1:Jsemerl8qK30jGNdYlxGZpZk9RjB4pqvezJgxqUgy30=
github.com/willoch/tago v0.0.0-20180311150625-8f2f8e8900dc/go.mod h1:9WHA/f8A/TRK+WQQZhqx47In4pnIhMTH6UrsgqqgsVQ=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/zeebo/assert v0.0.0-20181109011804-10f827ce2ed6/go.mod h1:yssERNPivllc1yU3BvpjYI5BUW+zglcz6QWqeVRL5t0=
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/blake3 v0.0.4-0.20200428182842-252974700486 h1:yh0zEy8it58x/IPNtKKuvKUkxSIaq4s5XiRSd40JuYs=
github.com/zeebo/blake3 v0.0.4-0.20200428182842-252974700486/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34=
github.com/zeebo/blake3 v0.0.4 h1:vtZ4X8B2lKXZFg2Xyg6Wo36mvmnJvc2VQYTtA4RDCkI=
github.com/zeebo/blake3 v0.0.4/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34=
github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05/go.mod h1:Gr+78ptB0MwXxm//LBaEvBiaXY7hXJ6KGe2V32X2F6E=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@ -197,6 +245,8 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU=
golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -215,6 +265,10 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8=
golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@ -238,6 +292,7 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

17
holder.go
View file

@ -115,6 +115,10 @@ type HolderOpts struct {
// If Inspect is set, we'll try to obtain additional information
// about fragments when opening them.
Inspect bool
// Txsrc controls the tx/storage engine we instatiate. Set by
// server.go OptServerTxsrc
Txsrc string
}
func (h *Holder) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) {
@ -507,6 +511,10 @@ func (h *Holder) Open() error {
if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
continue
}
// Skip badgerdb files too.
if strings.HasSuffix(fi.Name(), "badgerdb") {
continue
}
h.Logger.Printf("opening index: %s", filepath.Base(fi.Name()))
@ -614,8 +622,8 @@ func (h *Holder) Close() error {
}
// Begin starts a transaction on the holder.
func (h *Holder) Begin(writable bool) (Tx, error) {
return NewMultiTx(writable, h), nil
func (h *Holder) BeginTx(writable bool, index *Index) (Tx, error) {
return index.Txf.NewTx(Txo{Write: writable, Index: index}), nil
}
// HasData returns true if Holder contains at least one index.
@ -908,6 +916,11 @@ func (h *Holder) DeleteIndex(name string) error {
return errors.Wrap(err, "closing")
}
// remove any backing store.
if err := index.Txf.DeleteIndex(name); err != nil {
return errors.Wrap(err, "index.Txf.DeleteIndex")
}
// Delete index directory.
if err := os.RemoveAll(h.IndexPath(name)); err != nil {
return errors.Wrap(err, "removing directory")

15
holder_internal_test.go
View file

@ -79,21 +79,22 @@ func makeHolder() (*Holder, string, error) {
}
h := NewHolder(DefaultPartitionN)
h.Path = path
return h, h.Path, nil
return h, path, nil
}
func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) {
tx, err := h.Begin(true)
if err != nil {
t.Fatal(err)
}
defer func() { _ = tx.Rollback() }()
idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
if err != nil {
t.Fatalf("creating index: %v", err)
}
tx, err := h.BeginTx(writable, idx)
if err != nil {
t.Fatal(err)
}
defer tx.Rollback()
f, err := idx.CreateFieldIfNotExists(field, OptFieldTypeDefault())
if err != nil {
t.Fatalf("setting bit: %v", err)

96
holder_test.go
View file

@ -122,9 +122,14 @@ func TestHolder_Open(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
var idx *pilosa.Index
var err error
if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
}
if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -140,9 +145,13 @@ func TestHolder_Open(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
var idx *pilosa.Index
var err error
if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
}
if _, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if err := h.Holder.Close(); err != nil {
t.Fatal(err)
@ -162,15 +171,18 @@ func TestHolder_Open(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
tx, err := h.Begin(true)
var idx *pilosa.Index
var err error
if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
tx, err := h.BeginTx(writable, idx)
if err != nil {
t.Fatal(err)
}
defer func() { _ = tx.Rollback() }()
defer tx.Rollback()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := field.SetBit(tx, 0, 0, nil); err != nil {
t.Fatal(err)
@ -192,15 +204,19 @@ func TestHolder_Open(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
tx, err := h.Begin(true)
var idx *pilosa.Index
var err error
if idx, err = h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
tx, err := h.BeginTx(writable, idx)
if err != nil {
t.Fatal(err)
}
defer func() { _ = tx.Rollback() }()
defer tx.Rollback()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := field.SetBit(tx, 0, 0, nil); err != nil {
t.Fatal(err)
@ -220,15 +236,17 @@ func TestHolder_Open(t *testing.T) {
h := test.MustOpenHolder()
defer h.Close()
tx, err := h.Begin(true)
idx, err := h.CreateIndex("foo", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
defer func() { _ = tx.Rollback() }()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
tx, err := h.BeginTx(writable, idx)
if err != nil {
t.Fatal(err)
} else if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
}
defer tx.Rollback()
if field, err := idx.CreateField("bar", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := field.SetBit(tx, 0, 0, nil); err != nil {
t.Fatal(err)
@ -686,7 +704,10 @@ func TestHolderSyncer_IntField(t *testing.T) {
}
defer c.Close()
_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
var idx0 *pilosa.Index
_ = idx0
idx0, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
_ = idx0
if err != nil {
t.Fatalf("creating index i: %v", err)
}
@ -698,24 +719,40 @@ func TestHolderSyncer_IntField(t *testing.T) {
hldr0 := &test.Holder{Holder: c[0].Server.Holder()}
hldr1 := &test.Holder{Holder: c[1].Server.Holder()}
// Set data on the local holder for node0.
// Set data on the local holder for node0. columnID=1, value=1
hldr0.SetValue("i", "f", 1, 1)
// Set data on node1.
hldr1.SetValue("i", "f", 2, 2)
// in c0 expect the 1 bit
//idx0.Dump("in c0, before SyncData")
// Set data on node1. columnID=2, value=2
idx1 := hldr1.SetValue("i", "f", 2, 2)
_ = idx1
//idx1.Dump("in c1, before SyncData")
//vv("before c[0] SyncData")
err = c[0].Server.SyncData()
if err != nil {
t.Fatalf("syncing node 0: %v", err)
}
//vv("after c[0] SyncData")
// expect 3 rows, the 1 bit + 2 rows for the 2 value as BSI. But, we only see that c0 overwrote c1.
//idx0.Dump("in c0, after syncData")
//idx1.Dump("in c1, after syncData")
// Problem is: data at c1 was replaced by c0, instead of being merged with existing c1.
// Problem is: data at c0 did not receive and merge the c1 data.
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {
if a, exists := hldr.Value("i", "f", 1); !exists || a != 1 {
t.Errorf("unexpected value(node%d/0): %d, exists: %v", i, a, exists)
// expects exists==true, a==1
t.Errorf("unexpected value(node%d/0): a:%d, exists: %v", i, a, exists) // failing TestHolderSyncer_IntField under Badger, unexpected value(node1/0): a:0, exists: true
}
if a, exists := hldr.Value("i", "f", 2); exists {
t.Errorf("unexpected value(node%d/1): %d, exists: %v", i, a, exists)
t.Errorf("unexpected value(node%d/1): a:%d, exists: %v", i, a, exists)
}
}
})
@ -732,7 +769,10 @@ func TestHolderSyncer_IntField(t *testing.T) {
}
defer c.Close()
_, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
var idx0 *pilosa.Index
_ = idx0
idx0, err = c[0].API.CreateIndex(context.Background(), "i", pilosa.IndexOptions{})
_ = idx0
if err != nil {
t.Fatalf("creating index i: %v", err)
}
@ -769,6 +809,10 @@ func TestHolderSyncer_IntField(t *testing.T) {
t.Fatalf("syncing node 1: %v", err)
}
// dump the badger keys for both c0 and c1
//vv("in c0, allkeys = '%v'", idx0.StringifiedBadgerKeys(nil))
//vv("in c1, allkeys = '%v'", c[1].index.StringifiedBadgerKeys())
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {
if a := hldr.Range("i", "f", pql.GT, 0); !reflect.DeepEqual(a.Columns(), []uint64{2 * pilosa.ShardWidth, 3 * pilosa.ShardWidth, 4 * pilosa.ShardWidth}) {

36
index.go
View file

@ -67,16 +67,32 @@ type Index struct {
// Instantiates new translation stores
OpenTranslateStore OpenTranslateStoreFunc
// txf chooses the transaction and storage strategy
Txf *TxFactory
}
// NewIndex returns a new instance of Index.
func NewIndex(holder *Holder, path, name string) (*Index, error) {
err := validateName(name)
// Emulate what the spf13/cobra does, letting env vars override
// the defaults, because we may be under a simple "go test" run where
// not all that command line machinery has been spun up.
txsrc := os.Getenv("PILOSA_TXSRC")
if txsrc == "" {
txsrc = DefaultTxsrc
}
txf, err := newTxFactory(txsrc, path)
if err != nil {
return nil, errors.Wrap(err, "creating newTxFactory")
}
err = validateName(name)
if err != nil {
return nil, errors.Wrap(err, "validating name")
}
return &Index{
idx := &Index{
path: path,
name: name,
fields: make(map[string]*Field),
@ -94,7 +110,11 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
translationSyncer: NopTranslationSyncer,
OpenTranslateStore: OpenInMemTranslateStore,
}, nil
Txf: txf,
}
idx.Txf.idx = idx
return idx, nil
}
// CreatedAt is an timestamp for a specific version of an index.
@ -326,6 +346,11 @@ func (i *Index) Close() error {
i.mu.Lock()
defer i.mu.Unlock()
err := i.Txf.CloseIndex(i)
if err != nil {
return errors.Wrap(err, "closing index")
}
// Close the attribute store.
i.columnAttrs.Close()
@ -367,9 +392,8 @@ func (i *Index) AvailableShards() *roaring.Bitmap {
}
// Begin starts a transaction on a shard of the index.
func (i *Index) Begin(writable bool, shard uint64) (Tx, error) {
// TODO(bbj): Check for underlying storage as RBF or roaring.
return &RoaringTx{Index: i}, nil
func (i *Index) BeginTx(writable bool, shard uint64) (Tx, error) {
return i.Txf.NewTx(Txo{Write: writable, Index: i, Shard: shard}), nil
}
// fieldPath returns the path to a field in the index.

1
license.exceptions
View file

@ -9,3 +9,4 @@
./proto/pilosa.pb.go
./logger/filewriter.go
./logger/filewriter_test.go
./vprint.go

3
mmap_test.go
View file

@ -31,7 +31,8 @@ type cv struct {
func forceSnapshotsCheckMapping(t *testing.T) {
depth := uint(6)
f := mustOpenBSIFragment("i", "f", viewStandard, 0)
f, idx := mustOpenBSIFragment("i", "f", viewStandard, 0)
_ = idx
f.Logger = logger.NewLogfLogger(t)
defer f.Clean(t)

2
pql/ast.go
View file

@ -244,7 +244,7 @@ func (q *Query) WriteCallN() int {
var n int
for _, call := range q.Calls {
switch call.Name {
case "Set", "Clear", "SetRowAttrs", "SetColumnAttrs":
case "Set", "Clear", "SetRowAttrs", "SetColumnAttrs", "ClearRow", "Store", "SetBit":
n++
}
}

8
pql/decimal.go
View file

@ -68,6 +68,14 @@ type Decimal struct {
Scale int64
}
func (d Decimal) Clone() (r *Decimal) {
r = &Decimal{
Value: d.Value,
Scale: d.Scale,
}
return
}
// NewDecimal returns a Decimal based on the provided arguments.
func NewDecimal(value, scale int64) Decimal {
return Decimal{

2
roaring/containers_btree.go
View file

@ -210,6 +210,8 @@ type btcIterator struct {
val *Container
}
func (i *btcIterator) Close() {}
func (i *btcIterator) Next() bool {
k, v, err := i.e.Next()
if err == io.EOF {

7
roaring/containers_slice.go
View file

@ -91,11 +91,16 @@ func (sc *sliceContainers) GetOrCreate(key uint64) *Container {
}
func (sc *sliceContainers) Clone() Containers {
other := newSliceContainers()
other.keys = make([]uint64, len(sc.keys))
other.containers = make([]*Container, len(sc.containers))
copy(other.keys, sc.keys)
for i, c := range sc.containers {
if c == nil {
other.containers[i] = nil
continue
}
other.containers[i] = c.Clone()
}
return other
@ -234,6 +239,8 @@ type sliceIterator struct {
value *Container // current value
}
func (si *sliceIterator) Close() {}
func (si *sliceIterator) Next() bool {
if si.e == nil {
return false

154
roaring/roaring.go
View file

@ -174,6 +174,7 @@ type Containers interface {
type ContainerIterator interface {
Next() bool
Value() (uint64, *Container)
Close()
}
// Bitmap represents a roaring bitmap.
@ -235,6 +236,7 @@ var NewFileBitmap = NewBTreeBitmap
// Clone returns a heap allocated copy of the bitmap.
// Note: The OpWriter IS NOT copied to the new bitmap.
func (b *Bitmap) Clone() *Bitmap {
if b == nil {
return nil
}
@ -749,6 +751,8 @@ func (it *mutableContainersIterator) Value() (uint64, *Container) {
return it.cit.Value()
}
func (it *mutableContainersIterator) Close() {}
// IntersectInPlace returns the bitwise intersection of b and others,
// modifying b in place.
func (b *Bitmap) IntersectInPlace(others ...*Bitmap) {
@ -1716,10 +1720,10 @@ func (b *Bitmap) writeToUnoptimized(w io.Writer) (n int64, err error) {
return n, nil
}
// roaringIterator represents something which can iterate through a roaring
// RoaringIterator represents something which can iterate through a roaring
// bitmap and yield information about containers, including type, size, and
// the location of their data structures.
type roaringIterator interface {
type RoaringIterator interface {
// Len reports the number of containers total.
Len() (count int64)
// Next yields the information about the next container
@ -1728,6 +1732,19 @@ type roaringIterator interface {
// which is typically an ops log in our case, and also its offset in case
// we need to talk about truncation.
Remaining() ([]byte, int64)
// NextContainer is a helper that is used in place of Next(). It will
// allocate a Container from the output of its internal call to Next(),
// and return the key and container rc. If Next returns an error, then
// NextContainer will return 0, nil.
NextContainer() (key uint64, rc *Container)
// Data returns the underlying data, esp for the Ops log.
Data() []byte
// Clone copies the iterator, preserving it at this point in the iteration.
// It may well share much underlying data.
Clone() RoaringIterator
}
// baseRoaringIterator holds values used by both Pilosa and official Roaring
@ -1763,6 +1780,16 @@ func (b *baseRoaringIterator) SilenceLint() {
_ = b.prevOffset32
}
func (b *pilosaRoaringIterator) Clone() (clone RoaringIterator) {
cp := *b
return &cp
}
func (b *officialRoaringIterator) Clone() (clone RoaringIterator) {
cp := *b
return &cp
}
type pilosaRoaringIterator struct {
baseRoaringIterator
}
@ -1859,7 +1886,7 @@ func newPilosaRoaringIterator(data []byte) (*pilosaRoaringIterator, error) {
return r, nil
}
func newRoaringIterator(data []byte) (roaringIterator, error) {
func NewRoaringIterator(data []byte) (RoaringIterator, error) {
if len(data) < headerBaseSize {
return nil, errors.New("invalid data: not long enough to be a roaring header")
}
@ -1891,6 +1918,10 @@ func (r *baseRoaringIterator) Len() int64 {
return r.keys
}
func (r *baseRoaringIterator) Data() []byte {
return r.data
}
func (r *baseRoaringIterator) Remaining() ([]byte, int64) {
if r.lastDataOffset == 0 {
return nil, 0
@ -1898,6 +1929,20 @@ func (r *baseRoaringIterator) Remaining() ([]byte, int64) {
return r.data[r.lastDataOffset:], r.lastDataOffset
}
func (r *pilosaRoaringIterator) NextContainer() (key uint64, rc *Container) {
itrKey, itrCType, itrN, itrLen, itrPointer, itrErr := r.Next()
if itrErr != nil {
return 0, nil
}
rc = &Container{}
rc.typeID = itrCType
rc.n = int32(itrN)
rc.len = int32(itrLen)
rc.cap = int32(itrLen)
rc.pointer = itrPointer
return itrKey, rc
}
func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length int, pointer *uint16, err error) {
if r.currentIdx >= r.keys {
// we're already done
@ -1954,6 +1999,20 @@ func (r *pilosaRoaringIterator) Next() (key uint64, cType byte, n int, length in
return r.Current()
}
func (r *officialRoaringIterator) NextContainer() (key uint64, rc *Container) {
itrKey, itrCType, itrN, itrLen, itrPointer, itrErr := r.Next()
if itrErr != nil {
return 0, nil
}
rc = &Container{}
rc.typeID = itrCType
rc.n = int32(itrN)
rc.len = int32(itrLen)
rc.cap = int32(itrLen)
rc.pointer = itrPointer
return itrKey, rc
}
func (r *officialRoaringIterator) Next() (key uint64, cType byte, n int, length int, pointer *uint16, err error) {
if r.currentIdx >= r.keys {
// we're already done
@ -2066,7 +2125,7 @@ func (b *Bitmap) RemapRoaringStorage(data []byte) (mappedAny bool, returnErr err
if b.Containers == nil {
return false, nil
}
var itr roaringIterator
var itr RoaringIterator
var err error
var itrKey uint64
var itrCType byte
@ -2079,7 +2138,7 @@ func (b *Bitmap) RemapRoaringStorage(data []byte) (mappedAny bool, returnErr err
// map to the data. We still need to do the UpdateEvery loop, we
// just won't have an iterator for it.
if data != nil && b.preferMapping {
itr, err = newRoaringIterator(data)
itr, err = NewRoaringIterator(data)
}
// don't return early: we still have to do the unmapping
if err != nil {
@ -2144,7 +2203,16 @@ func (b *Bitmap) ImportRoaringBits(data []byte, clear bool, log bool, rowSize ui
if data == nil {
return 0, nil, errors.New("no roaring bitmap provided")
}
var itr roaringIterator
var itr RoaringIterator
itr, err = NewRoaringIterator(data)
if err != nil {
return 0, nil, err
}
return b.ImportRoaringRawIterator(itr, clear, log, rowSize)
}
func (b *Bitmap) ImportRoaringRawIterator(itr RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
var itrKey uint64
var itrCType byte
var itrN int
@ -2152,10 +2220,6 @@ func (b *Bitmap) ImportRoaringBits(data []byte, clear bool, log bool, rowSize ui
var itrPointer *uint16
var itrErr error
itr, err = newRoaringIterator(data)
if err != nil {
return 0, nil, err
}
if itr == nil {
return 0, nil, errors.New("failed to create roaring iterator, but don't know why")
}
@ -2225,7 +2289,7 @@ func (b *Bitmap) ImportRoaringBits(data []byte, clear bool, log bool, rowSize ui
}
err = nil
if log && changed > 0 {
op := op{opN: changed, roaring: data}
op := op{opN: changed, roaring: itr.Data()}
if clear {
op.typ = opTypeRemoveRoaring
} else {
@ -2257,13 +2321,13 @@ func (b *Bitmap) writeOp(op *op) error {
// Iterator returns a new iterator for the bitmap.
func (b *Bitmap) Iterator() *Iterator {
itr := &Iterator{bitmap: b}
itr := NewIterator(&BitmapIteratorFinder{b})
itr.Seek(0)
return itr
}
func (b *Bitmap) IteratorAt(start uint64) *Iterator {
itr := &Iterator{bitmap: b}
itr := NewIterator(&BitmapIteratorFinder{b})
itr.Seek(start)
return itr
}
@ -2287,7 +2351,7 @@ func RoaringToBitmaps(data []byte, shardWidth uint64) ([]*Bitmap, []uint64) {
if data == nil {
return nil, nil
}
var itr roaringIterator
var itr RoaringIterator
var itrKey uint64
var itrCType byte
var itrN int
@ -2300,7 +2364,7 @@ func RoaringToBitmaps(data []byte, shardWidth uint64) ([]*Bitmap, []uint64) {
var shards []uint64
keysPerShard := shardWidth >> 16
itr, err := newRoaringIterator(data)
itr, err := NewRoaringIterator(data)
if err != nil || itr == nil {
return nil, nil
}
@ -2524,15 +2588,38 @@ type BitmapInfo struct {
From, To uintptr // if set, indicates the address range used when unpacking
}
type IteratorFinder interface {
FindIterator(uint64) (ContainerIterator, bool)
Close()
}
type BitmapIteratorFinder struct {
bitmap *Bitmap
}
func (bif *BitmapIteratorFinder) FindIterator(seek uint64) (ContainerIterator, bool) {
return bif.bitmap.Containers.Iterator(seek)
}
func (bif *BitmapIteratorFinder) Close() {}
// Iterator represents an iterator over a Bitmap.
type Iterator struct {
bitmap *Bitmap
finder IteratorFinder
citer ContainerIterator
key uint64
c *Container
j, k int32 // i: container; j: array index, bit index, or run index; k: offset within the run
}
// NewIterator requires f as an IteratorFinder, it will
// crash if f is nil.
func NewIterator(f IteratorFinder) *Iterator {
return &Iterator{finder: f}
}
func (itr *Iterator) Close() {
itr.finder.Close()
}
// Seek moves to the first value equal to or greater than `seek`.
func (itr *Iterator) Seek(seek uint64) {
// k should always be -1 unless we're seeking into a run container. Then the
@ -2540,7 +2627,7 @@ func (itr *Iterator) Seek(seek uint64) {
itr.k = -1
// Move to the correct container.
itr.citer, _ = itr.bitmap.Containers.Iterator(highbits(seek))
itr.citer, _ = itr.finder.FindIterator(highbits(seek))
if !itr.citer.Next() {
itr.c = nil
return // eof
@ -6114,10 +6201,10 @@ func (b *Bitmap) BitwiseEqual(c *Bitmap) (bool, error) {
cn = biter.Next()
}
if bn {
return false, fmt.Errorf("container mismatch: %d vs %d containers, first bitmap has extra container %d [%d bits]", bct, cct, bk, bc)
return false, fmt.Errorf("container mismatch: %d vs %d containers, first bitmap has extra container %d [%v bits]", bct, cct, bk, bc)
}
if cn {
return false, fmt.Errorf("container mismatch: %d vs %d containers, second bitmap has extra container %d [%d bits]", bct, cct, ck, cc)
return false, fmt.Errorf("container mismatch: %d vs %d containers, second bitmap has extra container %d [%v bits]", bct, cct, ck, cc)
}
return true, nil
}
@ -6783,6 +6870,35 @@ func Optimize(c *Container) {
func Union(a, b *Container) *Container {
return union(a, b)
}
func Difference(a, b *Container) *Container {
return difference(a, b)
}
func (c *Container) Add(v uint16) (newC *Container, added bool) {
return c.add(v)
}
func (c *Container) Remove(v uint16) (c2 *Container, removed bool) {
return c.remove(v)
}
func (c *Container) Max() uint16 {
return c.max()
}
func (c *Container) CountRange(start, end int32) (n int32) {
return c.countRange(start, end)
}
func (c *Container) UnionInPlace(other *Container) *Container {
return c.unionInPlace(other)
}
func (c *Container) Difference(other *Container) *Container {
return difference(c, other)
}
func NewSliceContainers() *sliceContainers {
return newSliceContainers()
}

8
roaring/unmarshal_binary.go
View file

@ -27,7 +27,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) (err error) {
if data == nil {
return errors.New("no roaring bitmap provided")
}
var itr roaringIterator
var itr RoaringIterator
var itrKey uint64
var itrCType byte
var itrN int
@ -35,7 +35,7 @@ func (b *Bitmap) UnmarshalBinary(data []byte) (err error) {
var itrPointer *uint16
var itrErr error
itr, err = newRoaringIterator(data)
itr, err = NewRoaringIterator(data)
if err != nil {
return err
}
@ -110,7 +110,7 @@ func InspectBinary(data []byte, mapped bool, info *BitmapInfo) (b *Bitmap, mappe
if data == nil {
return b, mappedAny, errors.New("no roaring bitmap provided")
}
var itr roaringIterator
var itr RoaringIterator
var itrKey uint64
var itrCType byte
var itrN int
@ -118,7 +118,7 @@ func InspectBinary(data []byte, mapped bool, info *BitmapInfo) (b *Bitmap, mappe
var itrPointer *uint16
var itrErr error
itr, err = newRoaringIterator(data)
itr, err = NewRoaringIterator(data)
if err != nil {
return b, mappedAny, err
}

36
row.go
View file

@ -45,6 +45,40 @@ func NewRow(columns ...uint64) *Row {
return r
}
func (r *Row) Clone() (clone *Row) {
if r == nil {
return nil
}
var keyClone []string
if len(r.Keys) > 0 {
keyClone = make([]string, len(r.Keys))
copy(keyClone, r.Keys)
}
attrClone := make(map[string]interface{})
for k, v := range r.Attrs {
attrClone[k] = v
}
clone = &Row{
Keys: keyClone,
Attrs: attrClone,
}
for _, seg := range r.segments {
segClone := rowSegment{
shard: seg.shard,
writable: true, // we know it is safe; it is a copy.
n: seg.n,
}
if seg.data != nil {
segClone.data = seg.data.Clone() // *roaring.Bitmap
}
//segClone.InvalidateCount() // not needed?
clone.segments = append(clone.segments, segClone)
}
return clone
}
// NewRowFromBitmap divides a bitmap into rows, which it now calls shards. This
// transposes; data that was in any shard for Row 0 is now considered shard 0,
// etcetera.
@ -520,7 +554,7 @@ func (r *Row) MarshalJSON() ([]byte, error) {
func (r *Row) Columns() []uint64 {
a := make([]uint64, 0, r.Count())
for i := range r.segments {
a = append(a, r.segments[i].Columns()...)
a = append(a, r.segments[i].Columns()...) // Accessing Tx memory that is now invalid.
}
return a
}

11
server.go
View file

@ -326,6 +326,17 @@ func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption {
}
}
// OptServerTxsrc is a functional option on Server
// used to specify the transactional-storage to use,
// resulting in RoaringTx, RbfTx, BadgerTx, or a blueGreen* Tx
// being used for all Tx interface calls.
func OptServerTxsrc(txsrc string) ServerOption {
return func(s *Server) error {
s.holder.Opts.Txsrc = txsrc
return nil
}
}
// NewServer returns a new instance of Server.
func NewServer(opts ...ServerOption) (*Server, error) {
cluster := newCluster()

16
server/config.go
View file

@ -24,6 +24,7 @@ import (
"strings"
"time"
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/gossip"
"github.com/pilosa/pilosa/v2/toml"
"github.com/pkg/errors"
@ -166,6 +167,19 @@ type Config struct {
// MutexFraction is passed directly to runtime.SetMutexProfileFraction
MutexFraction int `toml:"mutex-fraction"`
} `toml:"profile"`
// Txsrc determines which Tx implementation the holder/Index will use; one
// of the available transactional-storage engines. Choices are listed
// in the string constants below. Should be one of
// "roaring","badger", "rbf", "badger_roaring", "roaring_badger", "rbf_roaring",
// "roaring_rbf", "badger_rbf", "rbf_badger", or any later addition. The
// engines with _ underscore indicate use of a blueGreenTx with a comparison
// of values back from each Tx method, and a panic if they differ. This
// is an effective test for consistency. If "rbf_roaring" is specified, then
// the roaring values are the ones actually returned from the blueGreenTx.
// If "roaring_rbf" is chosen, then the RBF values are the ones actually
// returned from the blueGreenTx.
Txsrc string `toml:"txsrc"`
}
// NewConfig returns an instance of Config with default options.
@ -222,6 +236,8 @@ func NewConfig() *Config {
c.Profile.BlockRate = 10000000 // 1 sample per 10 ms
c.Profile.MutexFraction = 100 // 1% sampling
c.Txsrc = pilosa.DefaultTxsrc
return c
}

23
server/handler_test.go
View file

@ -173,29 +173,38 @@ func TestHandler_Endpoints(t *testing.T) {
}
})
tx, err := holder.Begin(true)
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
tx0, err := holder.BeginTx(true, i0.Index)
if err != nil {
t.Fatal(err)
}
defer func() { _ = tx.Rollback() }()
defer tx0.Rollback()
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
tx1, err := holder.BeginTx(true, i1.Index)
if err != nil {
t.Fatal(err)
}
defer tx1.Rollback()
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx, 0, 0, nil); err != nil {
} else if _, err := f.SetBit(tx0, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(tx, 0, 0, nil); err != nil {
} else if _, err := f.SetBit(tx1, 0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.OptFieldTypeDefault()); err != nil {
t.Fatal(err)
}
if err := tx.Commit(); err != nil {
if err := tx0.Commit(); err != nil {
t.Fatal(err)
}
if err := tx1.Commit(); err != nil {
t.Fatal(err)
}
@ -638,7 +647,7 @@ func TestHandler_Endpoints(t *testing.T) {
t.Run("Query empty", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("")))
if body := w.Body.String(); body != `{"results":[]}`+"\n" {
if body := w.Body.String(); body != `{"results":[]}`+"\n" && body != `{"results":null}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})

60
test/holder.go
View file

@ -25,6 +25,8 @@ import (
"github.com/pilosa/pilosa/v2/pql"
)
var panicOn = pilosa.PanicOn
// Holder is a test wrapper for pilosa.Holder.
type Holder struct {
*pilosa.Holder
@ -86,29 +88,40 @@ func (h *Holder) Row(index, field string, rowID uint64) *pilosa.Row {
if err != nil {
panic(err)
}
tx := &pilosa.RoaringTx{Index: idx.Index}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: false, Index: idx.Index})
defer tx.Rollback()
row, err := f.Row(tx, rowID)
if err != nil {
panic(err)
}
return row
// clone it so that mmapped storage doesn't disappear from under it
// once the tx goes away.
return row.Clone()
}
// ReadRow returns a Row for a given field. If the field does not exist,
// it panics rather than creating the field.
func (h *Holder) ReadRow(index, field string, rowID uint64) *pilosa.Row {
f := h.Holder.Field(index, field)
idx := h.Holder.Index(index)
if idx == nil {
panic(pilosa.ErrIndexNotFound)
}
f := idx.Field(field)
if f == nil {
panic(pilosa.ErrFieldNotFound)
}
tx := &pilosa.RoaringTx{Field: f}
tx := idx.Txf.NewTx(pilosa.Txo{Write: false, Field: f})
defer tx.Rollback()
row, err := f.Row(tx, rowID)
if err != nil {
panic(err)
}
return row
// clone it so that mmapped storage doesn't disappear from under it
// once the tx goes away.
return row.Clone()
}
func (h *Holder) RowAttrStore(index, field string) pilosa.AttrStore {
@ -126,13 +139,17 @@ func (h *Holder) RowTime(index, field string, rowID uint64, t time.Time, quantum
if err != nil {
panic(err)
}
tx := &pilosa.RoaringTx{Index: idx.Index}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: false, Index: idx.Index})
defer tx.Rollback()
row, err := f.RowTime(tx, rowID, t, quantum)
if err != nil {
panic(err)
}
return row
// clone it so that mmapped storage doesn't disappear from under it
// once the tx goes away.
return row.Clone()
}
// SetBit sets a bit on the given field.
@ -147,12 +164,15 @@ func (h *Holder) SetBitTime(index, field string, rowID, columnID uint64, t *time
if err != nil {
panic(err)
}
tx := &pilosa.RoaringTx{Index: idx.Index}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index})
defer tx.Rollback()
_, err = f.SetBit(tx, rowID, columnID, t)
if err != nil {
panic(err)
}
panicOn(tx.Commit())
}
// ClearBit clears a bit on the given field.
@ -162,12 +182,14 @@ func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) {
if err != nil {
panic(err)
}
tx := &pilosa.RoaringTx{Index: idx.Index}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index})
defer tx.Rollback()
_, err = f.ClearBit(tx, rowID, columnID)
if err != nil {
panic(err)
}
panicOn(tx.Commit())
}
// MustSetBits sets columns on a row. Panic on error.
@ -179,18 +201,23 @@ func (h *Holder) MustSetBits(index, field string, rowID uint64, columnIDs ...uin
}
// SetValue sets an integer value on the given field.
func (h *Holder) SetValue(index, field string, columnID uint64, value int64) {
func (h *Holder) SetValue(index, field string, columnID uint64, value int64) *Index {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists(field, pilosa.OptFieldTypeInt(math.MinInt64, math.MaxInt64))
if err != nil {
panic(err)
}
tx := &pilosa.RoaringTx{Index: idx.Index}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: true, Index: idx.Index})
defer tx.Rollback()
_, err = f.SetValue(tx, columnID, value)
if err != nil {
panic(err)
}
if err := tx.Commit(); err != nil {
panic(err)
}
return idx
}
// Value returns the integer value for a given column.
@ -200,7 +227,8 @@ func (h *Holder) Value(index, field string, columnID uint64) (int64, bool) {
if err != nil {
panic(err)
}
tx := &pilosa.RoaringTx{Index: idx.Index}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: false, Index: idx.Index})
defer tx.Rollback()
val, exists, err := f.Value(tx, columnID)
if err != nil {
@ -217,11 +245,15 @@ func (h *Holder) Range(index, field string, op pql.Token, predicate int64) *pilo
if err != nil {
panic(err)
}
tx := &pilosa.RoaringTx{Index: idx.Index}
tx := idx.Index.Txf.NewTx(pilosa.Txo{Write: false, Index: idx.Index})
defer tx.Rollback()
row, err := f.Range(tx, field, op, predicate)
if err != nil {
panic(err)
}
return row
// clone it so that mmapped storage doesn't disappear from under it
// once the tx goes away.
return row.Clone()
}

344
tx.go
View file

@ -22,30 +22,160 @@ import (
"github.com/pkg/errors"
)
// batch operations want Tx.Add(batched=doBatch), while bit-at-a-time want Tx.Add(batched=!doBatched)
// Used in Tx.Add() to get consistency between RoaringTx and other Tx implementations on
// the changeCount returned.
const doBatched = false // must be false, do not change this without adjusting the Add() implementations correspondingly.
// writable initializes Tx that update, use !writable for read-only.
const writable = true
// Tx providers offer transactional storage for high-level roaring.Bitmaps and
// low-level roaring.Containers.
//
// The common 4-tuple of (index, field, view, shard) jointly specify a fragment.
// A fragment conceptually holds one roaring.Bitmap.
//
// Within the fragment, the ckey or container-key is the uint64 that specifies
// the high 48-bits of the roaring.Bitmap 64-bit space.
// The ckey is used to retreive a specific roaring.Container that
// is either a run, array, or raw-bitmap. The roaring.Container is the
// low 16-bits of the roaring.Bitmap space. Its size is at most
// 8KB (2^16 bits / (8 bits / byte) == 8192 bytes).
//
// The grain of the transaction is guaranteed to be at least at the shard
// within one index. Therefore updates to the any of the fields within
// the same shard will be atomically visible only once the transaction commits.
// Reads from another, concurrently open, transaction will not see updates
// that have not been committed.
type Tx interface {
Rollback() error
// Rollback must be called the end of read-only transactions. Either
// Rollback or Commit must be called at the end of writable transactions.
// It is safe to call Rollback multiple times, but it must be
// called at least once to release resources. Any Rollback after
// a Commit is ignored, so 'defer tx.Rollback()' should be commonly
// written after starting a new transaction.
//
// If there is an error during internal Rollback processing,
// this would be quite serious, and the underlying storage is
// expected to panic. Hence there is no explicit error returned
// from Rollback that needs to be checked.
Rollback()
// Commit makes the updates in the Tx visible to subsequent transactions.
Commit() error
// Readonly returns the flag this transaction was created with
// during NewTx. If the transaction is writable, it will return false.
Readonly() bool
// UseRowCache is used by fragment.go unprotectedRow() to determine
// dynamically at runtime if RoaringTx
// are in use, which for continuity want to continue to use the
// rowCache, or if other storage engines (RBF, Badger) are in
// use, which will mean that the bitmap data stored by the
// rowCache can disappear as it is un-mmap-ed, causing crashes.
UseRowCache() bool
// IncrementOpN updates internal statistics with the changedN provided.
IncrementOpN(index, field, view string, shard uint64, changedN int)
// Pointer gives us a memory address for the underlying
// transaction for debugging.
// It is public because we use it in roaring to report invalid
// container memory access outside of a transaction.
Pointer() string
// NewTxIterator returns it, a *roaring.Iterator whose it.Next() will
// successively return each uint64 stored in the conceptual roaring.Bitmap
// for the specified fragment.
NewTxIterator(index, field, view string, shard uint64) (it *roaring.Iterator)
// ContainerIterator loops over the containers in the conceptual
// roaring.Bitmap for the specified fragment.
// Calling Next() on the returned roaring.ContainerIterator gives
// you a roaring.Container that is either run, array, or raw bitmap.
// Return value 'found' is true when the ckey container was present.
ContainerIterator(index, field, view string, shard uint64, ckey uint64) (citer roaring.ContainerIterator, found bool, err error)
// RoaringBitmap retreives the roaring.Bitmap for the entire shard.
RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error)
Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error)
PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error
RemoveContainer(index, field, view string, shard uint64, key uint64) error
// Container returns the roaring.Container for the given ckey
// (container-key or highbits), in the chosen fragment.
Container(index, field, view string, shard uint64, ckey uint64) (*roaring.Container, error)
Add(index, field, view string, shard uint64, a ...uint64) (changed bool, err error)
Remove(index, field, view string, shard uint64, a ...uint64) (changed bool, err error)
// PutContainer stores c under the given ckey (container-key), in the specified fragment.
PutContainer(index, field, view string, shard uint64, ckey uint64, c *roaring.Container) error
// RemoveContainer deletes the roaring.Container under the given ckey (container-key),
// in the specified fragment.
RemoveContainer(index, field, view string, shard uint64, ckey uint64) error
// Add adds the 'a' bits to the specified fragment.
//
// Using batched=true allows efficient bulk-import.
//
// Notes on the RoaringTx implementation:
// If the batched flag is true, then the roaring.Bitmap.AddN() is used, which does oplog batches.
// If the batched flag is false, then the roaring.Bitmap.Add() is used, which does simple opTypeAdd single adds.
//
// Beware: if batched is true, then changeCount will only ever be 0 or 1,
// because it calls roaring.Add().
// If batched is false, we call roaring.DirectAddN() and then changeCount
// will be accurate if the changeCount is greater than 0.
//
// Hence: only ever call Add(batched=false) if changeCount is expected to be 0 or 1.
// Or, must use Add(batched=true) if changeCount can be > 1.
//
Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error)
// Remove removes the 'a' values from the Bitmap for the fragment.
Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error)
// Contains tests if the uint64 v is stored in the fragment's Bitmap.
Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error)
ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error)
// ForEach
ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error
// ForEachRange
ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error
// Count
Count(index, field, view string, shard uint64) (uint64, error)
// Max
Max(index, field, view string, shard uint64) (uint64, error)
// Min
Min(index, field, view string, shard uint64) (uint64, bool, error)
// UnionInPlace
UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error
// CountRange
CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error)
// OffsetRange
OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error)
// ImportRoaringBits does efficient bulk import using rit, a roaring.RoaringIterator.
// See the roaring package for details of the RoaringIterator.
// If clear is true, the bits from rit are cleared, otherwise they are set in the
// specifed fragment.
ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error)
}
// RawRoaringData used by ImportRoaringBits.
// must be consumable by roaring.newRoaringIterator()
type RawRoaringData struct {
data []byte
}
func (rr *RawRoaringData) Iterator() (roaring.RoaringIterator, error) {
return roaring.NewRoaringIterator(rr.data)
}
// MultiTx implements the transaction interface to combine multiple transactions.
@ -77,14 +207,40 @@ func NewMultiTxWithIndex(writable bool, index *Index) *MultiTx {
var _ Tx = (*MultiTx)(nil)
func (mtx *MultiTx) UseRowCache() bool {
return true
}
func (mtx *MultiTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
tx, err := mtx.tx(index, shard)
panicOn(err)
return tx.NewTxIterator(index, field, view, shard)
}
// Readonly is true if the transaction is not read-and-write, but only doing reads.
func (mtx *MultiTx) Readonly() bool {
return !mtx.writable
}
func (mtx *MultiTx) Pointer() string {
return fmt.Sprintf("%p", mtx)
}
func (tx *MultiTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
panic("not done")
}
func (mtx *MultiTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
tx, err := mtx.tx(index, shard)
panicOn(err)
tx.IncrementOpN(index, field, view, shard, changedN)
}
// Rollback rolls back all underlying transactions.
func (mtx *MultiTx) Rollback() (err error) {
func (mtx *MultiTx) Rollback() {
for _, tx := range mtx.txs {
if e := tx.Rollback(); e != nil && err == nil {
err = e
}
tx.Rollback()
}
return err
}
// Commit commits all underlying transactions.
@ -129,18 +285,18 @@ func (mtx *MultiTx) RemoveContainer(index, field, view string, shard uint64, key
return tx.RemoveContainer(index, field, view, shard, key)
}
func (mtx *MultiTx) Add(index, field, view string, shard uint64, a ...uint64) (changed bool, err error) {
func (mtx *MultiTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return false, err
return 0, err
}
return tx.Add(index, field, view, shard, a...)
return tx.Add(index, field, view, shard, batched, a...)
}
func (mtx *MultiTx) Remove(index, field, view string, shard uint64, a ...uint64) (changed bool, err error) {
func (mtx *MultiTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
tx, err := mtx.tx(index, shard)
if err != nil {
return false, err
return 0, err
}
return tx.Remove(index, field, view, shard, a...)
}
@ -231,8 +387,10 @@ func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) {
mtx.mu.Lock()
defer mtx.mu.Unlock()
mkey := multiTxKey{index: index, shard: shard, write: mtx.writable}
// Lookup transaction from cache.
tx := mtx.txs[multiTxKey{index, shard}]
tx := mtx.txs[mkey]
if tx != nil {
return tx, nil
}
@ -246,10 +404,10 @@ func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) {
}
// Begin tranaction & cache it.
if tx, err = idx.Begin(mtx.writable, shard); err != nil {
if tx, err = idx.BeginTx(mtx.writable, shard); err != nil {
return nil, err
}
mtx.txs[multiTxKey{index, shard}] = tx
mtx.txs[mkey] = tx
return tx, nil
}
@ -257,31 +415,66 @@ func (mtx *MultiTx) tx(index string, shard uint64) (_ Tx, err error) {
type multiTxKey struct {
index string
shard uint64
write bool
}
// RoaringTx represents a fake transaction object for Roaring storage.
type RoaringTx struct {
write bool
Index *Index
Field *Field
fragment *fragment
}
// Rollback is a no-op as Roaring does not support transactions.
func (tx *RoaringTx) Rollback() error {
return nil
func (tx *RoaringTx) UseRowCache() bool {
return true
}
func (tx *RoaringTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE
// the transaction Commits or Rollsback.
func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
b, err := tx.bitmap(index, field, view, shard)
panicOn(err)
return b.Iterator()
}
func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
b, err := tx.bitmap(index, field, view, shard)
panicOn(err)
if err != nil {
return 0, nil, err
}
return b.ImportRoaringRawIterator(rit, clear, true, rowSize)
}
func (tx *RoaringTx) Readonly() bool {
return !tx.write
}
func (tx *RoaringTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
frag, err := tx.getFragment(index, field, view, shard)
panicOn(err)
frag.incrementOpN(changedN)
}
// Rollback is a no-op as Roaring does not support transactions.
func (tx *RoaringTx) Rollback() {}
// Commit is a no-op as Roaring does not support transactions.
func (tx *RoaringTx) Commit() error {
return nil
}
func (tx *RoaringTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
return tx.bitmap(field, view, shard)
return tx.bitmap(index, field, view, shard)
}
func (tx *RoaringTx) Container(index, field, view string, shard uint64, key uint64) (*roaring.Container, error) {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, err
}
@ -289,7 +482,7 @@ func (tx *RoaringTx) Container(index, field, view string, shard uint64, key uint
}
func (tx *RoaringTx) PutContainer(index, field, view string, shard uint64, key uint64, c *roaring.Container) error {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
@ -298,7 +491,7 @@ func (tx *RoaringTx) PutContainer(index, field, view string, shard uint64, key u
}
func (tx *RoaringTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
@ -306,24 +499,48 @@ func (tx *RoaringTx) RemoveContainer(index, field, view string, shard uint64, ke
return nil
}
func (tx *RoaringTx) Add(index, field, view string, shard uint64, a ...uint64) (changed bool, err error) {
b, err := tx.bitmap(field, view, shard)
func (tx *RoaringTx) Add(index, field, view string, shard uint64, batched bool, a ...uint64) (changeCount int, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return false, err
return 0, err
}
return b.Add(a...)
if !batched {
changed, err := b.Add(a...)
if changed {
return 1, err
}
return 0, err
}
// Note: do not replace b.AddN() with b.DirectAddN().
// DirectAddN() does not do op-log operations inside roaring
// This creates a problem because RoaringTx needs the op-log
// to know when to flush the fragment to disk.
count, err := b.AddN(a...) // AddN does oplog batches. needed to keep op-log up to date.
return count, err
}
func (tx *RoaringTx) Remove(index, field, view string, shard uint64, a ...uint64) (changed bool, err error) {
b, err := tx.bitmap(field, view, shard)
func (tx *RoaringTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return false, err
return 0, err
}
return b.Remove(a...)
changed, err := b.Remove(a...) // green TestFragment_Bug_Q2DoubleDelete
panicOn(err)
if changed {
return 1, err
} else {
return 0, err
}
// Note: don't replace b.Remove(a...) with b.RemoveN(a...) or
// with b.DirectRemoveN(a...). If you do, you'll see
// TestFragment_Bug_Q2DoubleDelete go red.
}
func (tx *RoaringTx) Contains(index, field, view string, shard uint64, v uint64) (exists bool, err error) {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return false, err
}
@ -331,7 +548,7 @@ func (tx *RoaringTx) Contains(index, field, view string, shard uint64, v uint64)
}
func (tx *RoaringTx) ContainerIterator(index, field, view string, shard uint64, key uint64) (citer roaring.ContainerIterator, found bool, err error) {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, false, err
}
@ -340,7 +557,7 @@ func (tx *RoaringTx) ContainerIterator(index, field, view string, shard uint64,
}
func (tx *RoaringTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
@ -348,7 +565,7 @@ func (tx *RoaringTx) ForEach(index, field, view string, shard uint64, fn func(i
}
func (tx *RoaringTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
@ -356,7 +573,7 @@ func (tx *RoaringTx) ForEachRange(index, field, view string, shard uint64, start
}
func (tx *RoaringTx) Count(index, field, view string, shard uint64) (uint64, error) {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
@ -364,7 +581,7 @@ func (tx *RoaringTx) Count(index, field, view string, shard uint64) (uint64, err
}
func (tx *RoaringTx) Max(index, field, view string, shard uint64) (uint64, error) {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
@ -372,7 +589,7 @@ func (tx *RoaringTx) Max(index, field, view string, shard uint64) (uint64, error
}
func (tx *RoaringTx) Min(index, field, view string, shard uint64) (uint64, bool, error) {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, false, err
}
@ -381,7 +598,7 @@ func (tx *RoaringTx) Min(index, field, view string, shard uint64) (uint64, bool,
}
func (tx *RoaringTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
@ -390,7 +607,7 @@ func (tx *RoaringTx) UnionInPlace(index, field, view string, shard uint64, other
}
func (tx *RoaringTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return 0, err
}
@ -398,27 +615,43 @@ func (tx *RoaringTx) CountRange(index, field, view string, shard uint64, start,
}
func (tx *RoaringTx) OffsetRange(index, field, view string, shard uint64, offset, start, end uint64) (*roaring.Bitmap, error) {
b, err := tx.bitmap(field, view, shard)
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return nil, err
}
return b.OffsetRange(offset, start, end), nil
}
func (tx *RoaringTx) bitmap(field, view string, shard uint64) (*roaring.Bitmap, error) {
// getFragment is used by IncrementOpN() and by bitmap()
func (tx *RoaringTx) getFragment(index, field, view string, shard uint64) (*fragment, error) {
// If a fragment is attached, always use it.
if tx.fragment != nil {
return tx.fragment.storage, nil
return tx.fragment, nil
}
// If a field is attached, start from there.
// Otherwise look up the field from the index.
f := tx.Field
if f == nil {
if f = tx.Index.Field(field); f == nil {
return nil, ErrFieldNotFound
// we cannot assume that the tx.Index that we "started" on is the same
// as the index we are being queried; it might be foreign: TestExecutor_ForeignIndex
// So go through the holder
idx := tx.Index.holder.Index(index)
if idx == nil {
// only thing we can try is the cached index, and hope we aren't being asked for a foreign index.
f = tx.Index.Field(field)
if f == nil {
return nil, ErrFieldNotFound
}
} else {
if f = idx.Field(field); f == nil {
return nil, ErrFieldNotFound
}
}
}
// INVAR: f is not nil.
v := f.view(view)
if v == nil {
@ -429,5 +662,18 @@ func (tx *RoaringTx) bitmap(field, view string, shard uint64) (*roaring.Bitmap,
if frag == nil {
panic(fmt.Sprintf("fragment not found: %q / %q / %d", field, view, shard))
}
// Note: we cannot cache frag into tx.fragment.
// Empirically, it breaks 245 top-level pilosa tests.
// tx.fragment = frag // breaks the world.
return frag, nil
}
func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
frag, err := tx.getFragment(index, field, view, shard)
if err != nil {
return nil, err
}
return frag.storage, nil
}

469
txfactory.go Normal file
View file

@ -0,0 +1,469 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pkg/errors"
)
// public strings that pilosa/server/config.go can reference
const (
RoaringTxn string = "roaring"
BadgerTxn string = "badger"
RBFTxn string = "rbf"
// A is listed first, B is second. blueGreenTx returns the B output.
BlueGreenBadgerRoaring string = "badger_roaring"
BlueGreenRoaringBadger string = "roaring_badger"
BlueGreenRBFRoaring string = "rbf_roaring"
BlueGreenRoaringRBF string = "roaring_rbf"
BlueGreenBadgerRBF string = "badger_rbf"
BlueGreenRBFBadger string = "rbf_badger"
)
// DefaultTxsrc is set here. pilosa/server/config.go references it
// to set the default for pilosa server exeutable.
// Can be overridden with env variable PILOSA_TXSRC for testing.
const DefaultTxsrc = RoaringTxn
var sep = string(os.PathSeparator)
// TxFactory abstracts the creation of Tx interface-level
// transactions so that RBF, or Badger, or Roaring-fragment-files, or several
// of these at once in parallel, is used as the storage and transction layer.
type TxFactory struct {
typeOfTx txtype
bw *BadgerDBWrapper
// could have more than one *Index, but for now keep it simple,
// and allow blueGreenTx to report badger contents via idx
idx *Index
// TODO: put RBF database handle here.
}
// integer types for fast switch{}
type txtype int
const (
noneTxn txtype = 0
roaringFragmentFilesTxn txtype = 1 // these don't really have any transactions
badgerTxn txtype = 2
rbfTxn txtype = 3
// A is listed first, B is second. blueGreenTx returns the B output.
blueGreenBadgerRoaring txtype = 4
blueGreenRoaringBadger txtype = 5
blueGreenRBFRoaring txtype = 6
blueGreenRoaringRBF txtype = 7
blueGreenBadgerRBF txtype = 8
blueGreenRBFBadger txtype = 9
)
func txsrcToTxtype(txsrc string) txtype {
switch txsrc {
case RoaringTxn: // "roaring"
return roaringFragmentFilesTxn
case BadgerTxn: // "badger"
return badgerTxn
case RBFTxn: // "rbf"
return rbfTxn
case BlueGreenBadgerRoaring: //"badger_roaring"
return blueGreenBadgerRoaring
case BlueGreenRoaringBadger: // "roaring_badger"
return blueGreenRoaringBadger
case BlueGreenRBFRoaring: //"rbf_roaring"
return blueGreenRBFRoaring
case BlueGreenRoaringRBF: //"roaring_rbf"
return blueGreenRoaringRBF
case BlueGreenBadgerRBF: // "badger_rbf"
return blueGreenBadgerRBF
case BlueGreenRBFBadger: // "rbf_badger"
return blueGreenRBFBadger
}
panic(fmt.Sprintf("unknown txsrc '%v'", txsrc))
}
func newTxFactory(txsrc string, path string) (f *TxFactory, err error) {
ty := txsrcToTxtype(txsrc)
if ty < 1 || ty > 9 {
panic(fmt.Sprintf("invalid txtype '%v'", int(ty)))
}
var bw *BadgerDBWrapper
if ty == badgerTxn || ty == 4 || ty == 5 || ty == 8 || ty == 9 {
bw, err = openBadgerDBWrapper(path)
// TODO(jea): figure out what the appropriate error path is here.
//fmt.Printf("warning: could not open badgerdb on path '%v': '%v'. For safety, we are opening a new '%v-fallback' instead\n", path, err, path+"-fallback")
if err != nil {
//bw, err = newBadgerDBWrapper(path + "-fallback")
bw, err = newBadgerDBWrapper(path)
}
panicOn(err)
bw.doAllocZero = true
}
return &TxFactory{
typeOfTx: ty,
bw: bw,
}, err
}
// Txo holds the transaction options
type Txo struct {
Write bool
Field *Field
Index *Index
Fragment *fragment
Shard uint64
}
func (f *TxFactory) TxType() txtype {
return f.typeOfTx
}
func (f *TxFactory) DeleteIndex(name string) error {
switch f.typeOfTx {
case roaringFragmentFilesTxn:
// from holder.go:955, by default is already done there with os.RemoveAll()
return nil
case badgerTxn:
return f.bw.DeleteIndex(name)
case rbfTxn:
panic("todo rbfTxn DeleteIndex(name)")
case blueGreenBadgerRoaring:
return f.bw.DeleteIndex(name)
case blueGreenRoaringBadger:
return f.bw.DeleteIndex(name)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
func (f *TxFactory) Close() error {
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return nil
case badgerTxn:
// note cannot actually close Badger here.
// causes problems b/c tries holder.DeleteIndex tries to delete the index after db is closed.
//return f.bw.Close()
return nil
case rbfTxn:
panic("todo rbfTxn Close()")
case blueGreenBadgerRoaring:
return nil
case blueGreenRoaringBadger:
return nil
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
func (f *TxFactory) CloseIndex(idx *Index) error {
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return nil
case badgerTxn:
return nil
case rbfTxn:
panic("todo rbfTxn CloseIndex()")
case blueGreenBadgerRoaring:
return nil
case blueGreenRoaringBadger:
return nil
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
func (f *TxFactory) NewTx(o Txo) Tx {
switch f.typeOfTx {
case roaringFragmentFilesTxn:
return &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
case badgerTxn:
btx := f.bw.NewBadgerTx(o.Write)
return btx
case rbfTxn:
panic("todo rbfTxn creation")
case blueGreenBadgerRoaring:
btx := f.bw.NewBadgerTx(o.Write)
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(btx, rtx, f.idx)
case blueGreenRoaringBadger:
btx := f.bw.NewBadgerTx(o.Write)
rtx := &RoaringTx{write: o.Write, Field: o.Field, Index: o.Index, fragment: o.Fragment}
return newBlueGreenTx(rtx, btx, f.idx)
}
panic(fmt.Sprintf("unknown f.typeOfTx type: '%v'", f.typeOfTx))
}
func (ty txtype) String() string {
switch ty {
case noneTxn:
return "noneTxn"
case roaringFragmentFilesTxn:
return "roaringFragmentFilesTxn"
case badgerTxn:
return "badgerTxn"
case rbfTxn:
return "rbfTxn"
case blueGreenBadgerRoaring:
return "blueGreenBadgerRoaring"
case blueGreenRoaringBadger:
return "blueGreenRoaringBadger"
case blueGreenRBFRoaring:
return "blueGreenRBFRoaring"
case blueGreenRoaringRBF:
return "blueGreenRoaringRBF"
case blueGreenBadgerRBF:
return "blueGreenBadgerRBF"
case blueGreenRBFBadger:
return "blueGreenRBFBadger"
}
panic(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty)))
}
// StringifiedBadgerKeys displays the keys visible in BadgerDB for the idx *Index.
// If optionalUseThisTx is nil, it will start a new read-only transaction to
// do this query. Otherwise it will piggy back on the provided transaction.
// Hence to view uncommited keys, you must provide in optionalUseThisTx the
// Tx in which they have been added.
func (idx *Index) StringifiedBadgerKeys(optionalUseThisTx Tx) string {
return idx.Txf.bw.StringifiedBadgerKeys(optionalUseThisTx)
}
// fragmentSpecFromRoaringPath takes a path releative to the
// index directory, not including the name of the index itself.
// The path should not start with the path separator sep ('/' or '\\') rune.
func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64, err error) {
if len(path) == 0 {
err = fmt.Errorf("fragmentSpecFromRoaringPath error: path '%v' too short", path)
return
}
if path[:1] == sep {
err = fmt.Errorf("fragmentSpecFromRoaringPath error: path '%v' cannot start with separator '%v'; must be relative to the index base directory", path, sep)
return
}
// sample path:
// field view shard
// myfield/views/standard/fragments/0
s := strings.Split(path, "/")
n := len(s)
if n != 5 {
err = fmt.Errorf("len(s)=%v, but expected 5. path='%v'", n, path)
return
}
field = s[0]
view = s[2]
shard, err = strconv.ParseUint(s[4], 10, 64)
if err != nil {
err = fmt.Errorf("fragmentSpecFromRoaringPath(path='%v') could not parse shard '%v' as uint: '%v'", path, s[4], err)
}
return
}
func (idx *Index) StringifiedRoaringKeys() (r string) {
paths, err := listFilesUnderDir(idx.path, false, "", true)
panicOn(err)
index := idx.name
r = "allkeys:[\n"
for _, relpath := range paths {
field, view, shard, err := fragmentSpecFromRoaringPath(relpath)
if err != nil {
continue // ignore .meta paths
}
abspath := idx.path + sep + relpath
s, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard)
panicOn(err)
//r += fmt.Sprintf("path:'%v' fragment contains:\n") + s
r += s
}
r += "]\n all-in-blake3:" + blake3sum16([]byte(r)) + "\n"
return "roaring-" + r
}
func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64) (r string, err error) {
var info roaring.BitmapInfo
_ = info
var f *os.File
f, err = os.Open(path)
panicOn(err)
if err != nil {
return
}
var fi os.FileInfo
fi, err = f.Stat()
panicOn(err)
if err != nil {
return
}
// Memory map the file.
data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
err = errors.Wrap(err, "mmapping")
return
}
defer func() {
err := syscall.Munmap(data)
if err != nil {
panic(fmt.Errorf("loadRawRoaringContainer: munmap failed: %v", err))
}
panicOn(f.Close())
}()
// Attach the mmap file to the bitmap.
var rbm *roaring.Bitmap
rbm, _, err = roaring.InspectBinary(data, true, &info)
if err != nil {
err = errors.Wrap(err, "inspecting")
return
}
citer, found := rbm.Containers.Iterator(0)
_ = found // probably gonna use just the Ops log instead, so don't panic if !found.
for citer.Next() {
ckey, ct := citer.Value()
by := containerToBytes(ct)
hash := blake3sum16(by)
cts := roaring.NewSliceContainers()
cts.Put(ckey, ct)
rbm := &roaring.Bitmap{Containers: cts}
srbm := bitmapAsString(rbm)
panicOn(err)
bkey := string(badgerKey(index, field, view, shard, ckey))
r += fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, ct.N())
r += " ......." + srbm + "\n"
}
return
}
// listFilesUnderDir returns the paths of files found under directory root.
// If includeRoot is true, it returns the full path, otherwise paths are relative to root.
// If requriedSuffix is supplied, the returned file paths will end in that,
// and any other files found during the walk of the directory tree will be ignored.
// If ignoreEmpty is true, files of size 0 will be excluded.
func listFilesUnderDir(root string, includeRoot bool, requiredSuffix string, ignoreEmpty bool) (files []string, err error) {
if !dirExists(root) {
return nil, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root)
}
n := len(root) + 1
if includeRoot {
n = 0
}
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if len(path) < n {
// ignore
} else {
if info == nil {
panic(fmt.Sprintf("info was nil for path = '%v'", path))
}
if info.IsDir() {
// skip directories.
} else {
if ignoreEmpty && info.Size() == 0 {
return nil
}
if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) {
files = append(files, path[n:])
}
}
}
return nil
})
return
}
func dirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func fileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
var _ = fileSize // happy linter
// Dump prints to stdout the contents of the roaring Containers
// stored in idx. Its format may vary depending of the type of
// idx.Txf transaction factory that is in use.
// Mostly for debugging.
func (idx *Index) Dump(label string) {
ty := idx.Txf.TxType()
fileline := FileLine(2)
switch ty {
case badgerTxn:
fmt.Printf("%v Index.Dump('%v') for index '%v':\n%v\n", fileline, label, idx.name, idx.StringifiedBadgerKeys(nil))
return
case blueGreenRoaringBadger, blueGreenBadgerRoaring:
fmt.Printf("%v Index.Dump('%v') for index '%v', RoaringTx:\n%v\n", fileline, label, idx.name, idx.StringifiedRoaringKeys())
fmt.Printf("%v Index.Dump('%v') for index '%v', BadgerTx :\n%v\n", fileline, label, idx.name, idx.StringifiedBadgerKeys(nil))
return
case roaringFragmentFilesTxn:
fmt.Printf("%v Index.Dump('%v') for index '%v', BadgerTx :\n%v\n", fileline, label, idx.name, idx.StringifiedRoaringKeys())
return
}
panic(fmt.Errorf("%v Index.Dump('%v') for index '%v': no implementation for txtype '%v'\n", fileline, label, idx.name, ty))
}
func containerToBytes(ct *roaring.Container) []byte {
ty := roaring.ContainerType(ct)
switch ty {
case containerNil:
panic("nil container")
case containerArray:
return fromArray16(roaring.AsArray(ct))
case containerBitmap:
return fromArray64(roaring.AsBitmap(ct))
case containerRun:
return fromInterval16(roaring.AsRuns(ct))
}
panic(fmt.Sprintf("unknown container type '%v'", int(ty)))
}

6
utils_internal_test.go
View file

@ -135,11 +135,13 @@ func (t *ClusterCluster) SetBit(index, field string, rowID, colID uint64, x *tim
}
if err := func() error {
tx, err := c.holder.Begin(true)
tx, err := c.holder.BeginTx(writable, c.holder.indexes[f.index])
if tx != nil {
defer tx.Rollback()
}
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
if _, err := f.SetBit(tx, rowID, colID, x); err != nil {
return err

113
vprint.go Normal file
View file

@ -0,0 +1,113 @@
// home: https://github.com/glyerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package pilosa
import (
"fmt"
"io"
"os"
"path"
"runtime"
"runtime/debug"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("\n%s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}

2
xrbrsupport.go
View file

@ -59,7 +59,7 @@ func (rbc *RBFConverter) Convert(index, field, view string, shard uint64, rb *ro
if err != nil {
return err
}
tx, err := db.Begin(true)
tx, err := db.Begin(writable)
if err != nil {
return err
}