scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 17:15:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1553 from kuba--/wth-is-vprint

Browse source 
[CORE-452] Move `vprint` to separate package
This commit is contained in:
Kuba Podgórski 2021-03-29 18:10:18 +02:00 • committed by GitHub
commit b835221ded
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
52 changed files with 753 additions and 1799 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
api.go
View file

@ -1893,7 +1893,7 @@ func (api *API) GetTranslateEntryReader(ctx context.Context, offsets TranslateOf
defer func() {
if err != nil {
for i := range a {
a[i].Close()
a[i].Close() // nolint: errcheck
}
}
}()

27
api_test.go
View file

@ -30,6 +30,7 @@ import (
"github.com/pilosa/pilosa/v2/http"
"github.com/pilosa/pilosa/v2/server"
"github.com/pilosa/pilosa/v2/test"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
// attrFun defines a mapping from columnID -> attr value
@ -256,7 +257,7 @@ func TestAPI_Import(t *testing.T) {
if err := m0.API.Import(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
pql := fmt.Sprintf("Row(%s=%d)", fieldName, rowID)
@ -363,7 +364,7 @@ func TestAPI_ImportValue(t *testing.T) {
if err := coord.API.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
pql := fmt.Sprintf("Row(%s>0)", field)
@ -418,7 +419,7 @@ func TestAPI_ImportValue(t *testing.T) {
if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
query := fmt.Sprintf("Row(%s>6)", field)
// Query node0.
if res, err := m0.API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: query}); err != nil {
@ -489,7 +490,7 @@ func TestAPI_ImportValue(t *testing.T) {
if err := m0.API.ImportValue(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
pql := fmt.Sprintf(`Row(%s=="strval-110")`, field)
@ -579,12 +580,12 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
bitIsSet := func() bool {
query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID)
res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
panicOn(err)
PanicOn(err)
cols := res.Results[0].(*pilosa.Row).Columns()
for i := range cols {
if cols[i] == acctOwnerID {
@ -595,13 +596,13 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
}
if !bitIsSet() {
panic("IRA bit should have been set")
PanicOn("IRA bit should have been set")
}
queryAcct := func(m0api *pilosa.API, acctOwnerID uint64, fieldAcct0, index string) (acctBal int64) {
query := fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct0, acctOwnerID)
res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
panicOn(err)
PanicOn(err)
if len(res.Results) == 0 {
return 0
@ -613,7 +614,7 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
bal := queryAcct(m0api, acctOwnerID, fieldAcct0, index)
if bal != acct0bal {
panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, bal))
PanicOn(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, bal))
}
// clear the bit
@ -622,10 +623,10 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
if err := m0api.Import(ctx, qcx, ir0); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
if bitIsSet() {
panic("IRA bit should have been cleared")
PanicOn("IRA bit should have been cleared")
}
// clear the BSI
@ -634,11 +635,11 @@ func TestAPI_ClearFlagForImportAndImportValues(t *testing.T) {
if err := m0api.ImportValue(ctx, qcx, ivr0); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
bal = queryAcct(m0api, acctOwnerID, fieldAcct0, index)
if bal != 0 {
panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, 0))
PanicOn(fmt.Sprintf("expected %v, observed %v starting acct0 balance", acct0bal, 0))
}
}

152
bluegreentx.go
View file

@ -23,7 +23,7 @@ import (
"github.com/pilosa/pilosa/v2/roaring"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
//txkey "github.com/pilosa/pilosa/v2/txkey"
. "github.com/pilosa/pilosa/v2/vprint"
)
// blueGreenTx runs two Tx together and notices differences in their output.
@ -129,7 +129,7 @@ func (b *blueGreenRegistry) Close() {
b.mu.Lock()
defer b.mu.Unlock()
if len(b.m) > 0 {
panic(fmt.Sprintf("still have open/unchecked blueGreenTx: '%#v'", b.m))
PanicOn(fmt.Sprintf("still have open/unchecked blueGreenTx: '%#v'", b.m))
//AlwaysPrintf("still have unchecked blueGreenTx: '%#v'", b.m)
}
}
@ -191,7 +191,7 @@ func (c *blueGreenTx) Readonly() bool {
a := c.a.Readonly()
b := c.b.Readonly()
if a != b {
panic(fmt.Sprintf("Readonly difference, a=%v, but b =%v", a, b))
PanicOn(fmt.Sprintf("Readonly difference, a=%v, but b =%v", a, b))
}
return b
}
@ -240,41 +240,41 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
if aFound != bFound {
c.Dump(c.short, shard)
panic(fmt.Sprintf("compareTxState[%v]: A(%v) ContainerIterator had aFound=%v, but B(%v) had bFound=%v; at '%v'", here, c.as, aFound, c.bs, bFound, stack()))
PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) ContainerIterator had aFound=%v, but B(%v) had bFound=%v; at '%v'", here, c.as, aFound, c.bs, bFound, Stack()))
}
if aErr != nil || bErr != nil {
if aErr != nil && bErr != nil {
c.Dump(c.short, shard)
panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err '%v'; B(%v) reported err '%v' at %v", here, c.as, aErr, c.bs, bErr, stack()))
PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) reported err '%v'; B(%v) reported err '%v' at %v", here, c.as, aErr, c.bs, bErr, Stack()))
}
if aErr != nil {
c.Dump(c.short, shard)
panic(fmt.Sprintf("compareTxState[%v]: A(%v) reported err %v at %v; but B(%v) did not", here, c.as, aErr, c.bs, stack()))
PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) reported err %v at %v; but B(%v) did not", here, c.as, aErr, c.bs, Stack()))
}
if bErr != nil {
c.Dump(c.short, shard)
panic(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, stack()))
PanicOn(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, Stack()))
}
}
for aIter.Next() {
aKey, aValue := aIter.Value()
if !bIter.Next() {
AlwaysPrintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, dump to follow, stack=\n %v\n\n and here is dump:", here, c.as, aKey, c.bs, stack())
AlwaysPrintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, dump to follow, Stack=\n %v\n\n and here is dump:", here, c.as, aKey, c.bs, Stack())
c.Dump(c.short, shard)
panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, stack()))
PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, Stack()))
}
bKey, bValue := bIter.Value()
if bKey != aKey {
AlwaysPrintf("problem in caller %v", Caller(2))
c.Dump(c.short, shard)
panic(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, stack()))
PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, Stack()))
}
if err := aValue.BitwiseCompare(bValue); err != nil {
c.Dump(c.short, shard)
//vv("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack())
panic(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at stack=%v", here, aKey, err, c.as, c.bs, stack()))
//vv("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at Stack=%v", here, aKey, err, c.as, c.bs, Stack())
PanicOn(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at Stack=%v", here, aKey, err, c.as, c.bs, Stack()))
}
//vv("successfully matched aKey(%v)='%v' and bKey(%v)='%v'", c.as, aKey, c.bs, bKey)
}
@ -283,7 +283,7 @@ func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) {
AlwaysPrintf("bIter has more than it should. problem in caller %v. _sn_ %v", Caller(2), c.Sn())
c.Dump(c.short, shard)
bKey, _ := bIter.Value()
panic(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, (a.sn=%v) (b.sn=%v) at %v", here, c.bs, bKey, c.as, c.a.Sn(), c.b.Sn(), stack()))
PanicOn(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, (a.sn=%v) (b.sn=%v) at %v", here, c.bs, bKey, c.as, c.a.Sn(), c.b.Sn(), Stack()))
}
//vv("done without problem. compareTxState here = '%v', _sn_ %v gid=%v", here, c.Sn(), curGID())
}
@ -341,7 +341,7 @@ func (c *blueGreenTx) Rollback() {
c.mu.Lock()
defer c.mu.Unlock()
if c.rollbackOrCommitDone {
return // avoid using discarded tx for Dump, which will panic.
return // avoid using discarded tx for Dump, which will PanicOn.
}
c.rollbackOrCommitDone = true
@ -350,8 +350,8 @@ func (c *blueGreenTx) Rollback() {
}
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
//vv("blueGreenTx.Rollback() about to call (%v) a.Rollback()", c.as)
@ -379,8 +379,8 @@ func (c *blueGreenTx) Commit() error {
}
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
errA := c.a.Commit()
@ -396,8 +396,8 @@ func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*r
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
a, errA := c.a.RoaringBitmap(index, field, view, shard)
@ -409,7 +409,7 @@ func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*r
slcA := a.Slice()
slcB := b.Slice()
if !reflect.DeepEqual(slcA, slcB) {
panic("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!")
PanicOn("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!")
}
}
return b, errB
@ -419,8 +419,8 @@ func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uin
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
a, errA := c.a.Container(index, field, view, shard, key)
@ -429,7 +429,7 @@ func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uin
if !c.o.blueGreenOff {
compareErrors(errA, errB)
err = a.BitwiseCompare(b)
panicOn(err)
PanicOn(err)
}
return b, errB
}
@ -438,8 +438,8 @@ func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
errA := c.a.PutContainer(index, field, view, shard, key, rc)
@ -463,14 +463,14 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64,
// ================== end save comments.
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
// remember where the iterator started, so we can replay it a second time.
rit2 := rit.Clone()
panicOn(err)
PanicOn(err)
changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize, data)
@ -482,18 +482,18 @@ func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64,
// case where we know that RoaringTx.ImportRoaringBits changed and rowSet will
// be inaccurate.
if changedA != changedB {
panic(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB))
PanicOn(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB))
}
if len(rowSetA) != len(rowSetB) {
panic(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB))
PanicOn(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB))
}
for k, va := range rowSetA {
vb, ok := rowSetB[k]
if !ok {
panic(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB))
PanicOn(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB))
}
if va != vb {
panic(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb))
PanicOn(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb))
}
}
}
@ -507,8 +507,8 @@ func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, k
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
errA := c.a.RemoveContainer(index, field, view, shard, key)
@ -531,9 +531,9 @@ var _ = (&blueGreenTx{}).isIn // happy linter
func (c *blueGreenTx) isIn(index, field, view string, shard uint64, ckey uint64) (r []bool) {
r = make([]bool, 2)
inA, errA := c.a.Contains(index, field, view, shard, ckey)
panicOn(errA)
PanicOn(errA)
inB, errB := c.b.Contains(index, field, view, shard, ckey)
panicOn(errB)
PanicOn(errB)
r[0] = inA
r[1] = inB
return
@ -544,8 +544,8 @@ func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool,
//vv("blueGreenTx) Add(index=%v, field=%v, view=%v, shard=%v", index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Add() panic '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, stack())
panic(r)
AlwaysPrintf("see Add() PanicOn '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, Stack())
PanicOn(r)
}
}()
@ -562,7 +562,7 @@ func (c *blueGreenTx) Add(index, field, view string, shard uint64, batched bool,
if !c.o.blueGreenOff {
if ach != bch {
panic(fmt.Sprintf("Add() difference, ach=%v, but bch=%v; errA='%v'; errB='%v'", ach, bch, errA, errB))
PanicOn(fmt.Sprintf("Add() difference, ach=%v, but bch=%v; errA='%v'; errB='%v'", ach, bch, errA, errB))
}
compareErrors(errA, errB)
}
@ -575,14 +575,14 @@ func compareErrors(errA, errB error) {
case errA == nil && errB == nil:
// OK
case errA == nil:
panic(fmt.Sprintf("errA is nil, but errB = %#v", errB))
PanicOn(fmt.Sprintf("errA is nil, but errB = %#v", errB))
case errB == nil:
panic(fmt.Sprintf("errB is nil, but errA = %#v", errA))
PanicOn(fmt.Sprintf("errB is nil, but errA = %#v", errA))
default:
ae := errA.Error()
be := errB.Error()
if ae != be {
panic(fmt.Sprintf("errA is '%v', but errB is '%v'", ae, be))
PanicOn(fmt.Sprintf("errA is '%v', but errB is '%v'", ae, be))
}
}
}
@ -591,8 +591,8 @@ func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint6
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
ach, errA := c.a.Remove(index, field, view, shard, a...)
@ -609,8 +609,8 @@ func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
ax, errA := c.a.Contains(index, field, view, shard, key)
@ -627,8 +627,8 @@ func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64,
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
@ -690,7 +690,7 @@ func (bgi *blueGreenIterator) Next() bool {
na := bgi.ait.Next()
nb := bgi.bit.Next()
if na != nb {
panic(fmt.Sprintf("na=%v(%v) != nb(%v)=%v", na, bgi.as, bgi.bs, nb))
PanicOn(fmt.Sprintf("na=%v(%v) != nb(%v)=%v", na, bgi.as, bgi.bs, nb))
}
return nb
}
@ -701,10 +701,10 @@ func (bgi *blueGreenIterator) Value() (uint64, *roaring.Container) {
if !bgi.tx.o.blueGreenOff {
if ka != kb {
panic(fmt.Sprintf("ka=%v != kb=%v", ka, kb))
PanicOn(fmt.Sprintf("ka=%v != kb=%v", ka, kb))
}
err := ca.BitwiseCompare(cb)
panicOn(err)
PanicOn(err)
}
return kb, cb
}
@ -718,8 +718,8 @@ func (bgi *blueGreenIterator) Close() {
func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.ForEach(index, field, view, shard, fn)
@ -733,8 +733,8 @@ func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, star
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
@ -748,8 +748,8 @@ func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, er
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
a, errA := c.a.Count(index, field, view, shard)
@ -767,8 +767,8 @@ func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, erro
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
a, errA := c.a.Max(index, field, view, shard)
@ -786,8 +786,8 @@ func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
amin, afound, errA := c.a.Min(index, field, view, shard)
@ -805,8 +805,8 @@ func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, othe
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
errA := c.a.UnionInPlace(index, field, view, shard, others...)
@ -822,8 +822,8 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start,
defer func() {
if r := recover(); r != nil {
c.Dump(c.short, shard)
AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
a, errA := c.a.CountRange(index, field, view, shard, start, end)
@ -831,7 +831,7 @@ func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start,
if !c.o.blueGreenOff {
if a != b {
panic(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b))
PanicOn(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b))
}
compareErrors(errA, errB)
@ -843,8 +843,8 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star
c.checker.see(index, field, view, shard)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see OffsetRange() on _sn_ %v, panic '%v' at '%v'", c.Sn(), r, stack())
panic(r)
AlwaysPrintf("see OffsetRange() on _sn_ %v, PanicOn '%v' at '%v'", c.Sn(), r, Stack())
PanicOn(r)
}
}()
a, errA := c.a.OffsetRange(index, field, view, shard, offset, start, end)
@ -855,7 +855,7 @@ func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, star
err = roaringBitmapDiff(a, b)
if err != nil {
c.Dump(false, shard)
panicOn(fmt.Errorf("on _sn_ %v OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) err: %v", c.Sn(), index, field, view, int(shard), offset, start, end, err))
PanicOn(fmt.Errorf("on _sn_ %v OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) err: %v", c.Sn(), index, field, view, int(shard), offset, start, end, err))
}
compareErrors(errA, errB)
}
@ -868,8 +868,8 @@ func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint6
defer func() {
if r := recover(); r != nil {
c.Dump(c.short, shard)
AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
@ -891,7 +891,7 @@ func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint6
}
if sizeMustMatch {
if szA != szB {
panic(fmt.Sprintf("szA(%v) = %v, but szB(%v) = %v; fragmentPathForRoaring='%v'", c.as, szA, c.bs, szB, fragmentPathForRoaring))
PanicOn(fmt.Sprintf("szA(%v) = %v, but szB(%v) = %v; fragmentPathForRoaring='%v'", c.as, szA, c.bs, szB, fragmentPathForRoaring))
}
return &MultiReaderB{a: rcA, b: rcB}, szB, errB
} else {
@ -955,14 +955,14 @@ func (m *MultiReaderB) Read(p []byte) (nB int, errB error) {
if !m.allowSizeVariation {
if errA == io.ErrUnexpectedEOF {
panic(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA))
PanicOn(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA))
}
if nA != nB {
panic(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA))
PanicOn(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA))
}
cmp := bytes.Compare(p[:nB], p2[:nB])
if cmp != 0 {
panic(fmt.Sprintf("MultiReaderB reads p and p2 (cmp= %v) differed.", cmp))
PanicOn(fmt.Sprintf("MultiReaderB reads p and p2 (cmp= %v) differed.", cmp))
}
}
return

6
bluegreentx_test.go
View file

@ -24,6 +24,8 @@ import (
"testing"
cryrand "crypto/rand"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
var _ = context.Background
@ -42,7 +44,7 @@ func TestMultiReaderB(t *testing.T) {
nr := 0
for nr < n {
na, err := src.Read(a)
panicOn(err)
PanicOn(err)
nr += na
}
if nr != n {
@ -62,7 +64,7 @@ func TestMultiReaderB(t *testing.T) {
// should not trigger the internal panic of MultiReadB
ncp, err := io.Copy(ioutil.Discard, m)
panicOn(err)
PanicOn(err)
if ncp != int64(n) {
panic("short copy")
}

33
bolt.go
View file

@ -31,6 +31,7 @@ import (
"github.com/pilosa/pilosa/v2/hash"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/storage"
. "github.com/pilosa/pilosa/v2/vprint"
// On Bolt only, we still use the long txkey, because
// this allows Max() to work readily.
@ -132,7 +133,7 @@ func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storag
dir := filepath.Dir(path)
if !DirExists(path) {
panicOn(os.MkdirAll(dir, 0755))
PanicOn(os.MkdirAll(dir, 0755))
}
db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize})
@ -522,7 +523,7 @@ func (tx *BoltTx) Commit() error {
defer tx.mu.Unlock()
err := tx.tx.Commit()
panicOn(err)
PanicOn(err)
tx.o.dbs.Cleanup(tx)
return err
@ -657,15 +658,15 @@ func (tx *BoltTx) addOrRemove(index, field, view string, shard uint64, batched,
// not first time through, write what we got.
if remove && (rc == nil || rc.N() == 0) {
err = tx.RemoveContainer(index, field, view, shard, lastHi)
panicOn(err)
PanicOn(err)
} else {
err = tx.PutContainer(index, field, view, shard, lastHi, rc)
panicOn(err)
PanicOn(err)
}
}
// get the next container
rc, err = tx.Container(index, field, view, shard, hi)
panicOn(err)
PanicOn(err)
} // else same container, keep adding bits to rct.
chng := false
// rc can be nil before, and nil after, in both Remove/Add below.
@ -684,17 +685,17 @@ func (tx *BoltTx) addOrRemove(index, field, view string, shard uint64, batched,
if remove {
if rc == nil || rc.N() == 0 {
err = tx.RemoveContainer(index, field, view, shard, hi)
panicOn(err)
PanicOn(err)
} else {
err = tx.PutContainer(index, field, view, shard, hi, rc)
panicOn(err)
PanicOn(err)
}
} else {
if rc == nil || rc.N() == 0 {
panic("there should be no way to have an empty bitmap AFTER an Add() operation")
}
err = tx.PutContainer(index, field, view, shard, hi, rc)
panicOn(err)
PanicOn(err)
}
return
}
@ -955,7 +956,7 @@ type boltFinder struct {
// FindIterator lets boltFinder implement the roaring.FindIterator interface.
func (bf *boltFinder) FindIterator(seek uint64) (roaring.ContainerIterator, bool) {
a, found, err := bf.tx.ContainerIterator(bf.index, bf.field, bf.view, bf.shard, seek)
panicOn(err)
PanicOn(err)
bf.needClose = append(bf.needClose, a)
return a, found
}
@ -1015,7 +1016,7 @@ func (tx *BoltTx) ForEachRange(index, field, view string, shard uint64, start, e
func (tx *BoltTx) Count(index, field, view string, shard uint64) (uint64, error) {
a, found, err := tx.ContainerIterator(index, field, view, shard, 0)
panicOn(err)
PanicOn(err)
defer a.Close()
if !found {
return 0, nil
@ -1101,7 +1102,7 @@ func (tx *BoltTx) Min(index, field, view string, shard uint64) (uint64, bool, er
func (tx *BoltTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
rbm, err := tx.RoaringBitmap(index, field, view, shard)
panicOn(err)
PanicOn(err)
rbm.UnionInPlace(others...)
// iterate over the containers that changed within rbm, and write them back to disk.
@ -1115,7 +1116,7 @@ func (tx *BoltTx) UnionInPlace(index, field, view string, shard uint64, others .
// TODO: only write the changed ones back, as optimization?
// Compare to ImportRoaringBits.
err := tx.PutContainer(index, field, view, shard, containerKey, rc)
panicOn(err)
PanicOn(err)
}
return nil
}
@ -1133,7 +1134,7 @@ func (tx *BoltTx) CountRange(index, field, view string, shard uint64, start, end
citer, found, err := tx.ContainerIterator(index, field, view, shard, skey)
_ = found
panicOn(err)
PanicOn(err)
defer citer.Close()
@ -1265,7 +1266,7 @@ func (tx *BoltTx) ImportRoaringBits(index, field, view string, shard uint64, itr
// INVAR: nsynth > 0
oldC, err = tx.Container(index, field, view, shard, itrKey)
panicOn(err)
PanicOn(err)
if err != nil {
return
}
@ -1346,7 +1347,7 @@ func (tx *BoltTx) ImportRoaringBits(index, field, view string, shard uint64, itr
err = tx.PutContainer(index, field, view, shard, itrKey, newC)
if err != nil {
panicOn(err)
PanicOn(err)
return
}
continue
@ -1567,7 +1568,7 @@ func (w *BoltWrapper) DeletePrefix(prefix []byte) error {
w.muDb.Unlock()
err := tx.Commit()
panicOn(err)
PanicOn(err)
return nil
}

201
bolt_test.go
View file

@ -21,6 +21,7 @@ import (
"testing"
"github.com/pilosa/pilosa/v2/roaring"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
// helpers, each runs their own new txn, and commits if a change/delete
@ -31,7 +32,7 @@ func BoltMustHaveBitvalue(dbwrap *BoltWrapper, index, field, view string, shard
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
PanicOn(err)
if !exists {
panic(fmt.Sprintf("ARG bitvalue '%v' was NOT SET!!!", bitvalue))
}
@ -44,7 +45,7 @@ func BoltMustNotHaveBitvalue(dbwrap *BoltWrapper, index, field, view string, sha
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
PanicOn(err)
if exists {
panic(fmt.Sprintf("ARG bitvalue '%v' WAS SET but should not have been.!!!", bitvalue))
}
@ -59,36 +60,36 @@ func BoltMustSetBitvalue(dbwrap *BoltWrapper, index, field, view string, shard u
if changed != 1 {
panic("should have 1 bit changed")
}
panicOn(err)
PanicOn(err)
exists, err := tx.Contains(index, field, view, shard, putme)
panicOn(err)
PanicOn(err)
if !exists {
panic("ARG putme was NOT SET!!!")
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
}
func BoltMustDeleteBitvalueContainer(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) {
tx, _ := dbwrap.NewTx(writable, index, Txo{})
hi := highbits(putme)
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
panicOn(tx.Commit())
PanicOn(tx.RemoveContainer(index, field, view, shard, hi))
PanicOn(tx.Commit())
}
func BoltMustDeleteBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) {
tx, _ := dbwrap.NewTx(writable, index, Txo{})
_, err := tx.Remove(index, field, view, shard, putme)
panicOn(err)
panicOn(tx.Commit())
PanicOn(err)
PanicOn(tx.Commit())
}
func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) {
var err error
fn := path
panicOn(os.RemoveAll(fn))
PanicOn(os.RemoveAll(fn))
ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, nil)
panicOn(err)
PanicOn(err)
w = ww.(*BoltWrapper)
// verify it is empty
@ -99,7 +100,7 @@ func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) {
return w, func() {
w.Close()
panicOn(os.RemoveAll(fn))
PanicOn(os.RemoveAll(fn))
}
}
@ -126,28 +127,28 @@ func TestBolt_DeleteFragment(t *testing.T) {
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
PanicOn(err)
}
}
for _, view := range views {
for _, v := range bits {
exists, err := tx.Contains(index, field, view, shard, v)
panicOn(err)
PanicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!!")
}
}
}
err := tx.Commit()
panicOn(err)
PanicOn(err)
// end of setup
victim := "v1"
survivor := "v2"
err = dbwrap.DeleteFragment(index, field, victim, shard, nil)
panicOn(err)
PanicOn(err)
tx, _ = dbwrap.NewTx(!writable, index, Txo{})
defer tx.Rollback()
@ -155,7 +156,7 @@ func TestBolt_DeleteFragment(t *testing.T) {
for _, view := range views {
for _, v := range bits {
exists, err := tx.Contains(index, field, view, shard, v)
panicOn(err)
PanicOn(err)
if view == survivor {
if !exists {
panic(fmt.Sprintf("ARG survivor died : bit %v", v))
@ -209,7 +210,7 @@ func TestBolt_Max_on_many_containers(t *testing.T) {
for _, shard := range shards {
max, err := tx.Max(index, field, view, uint64(shard))
panicOn(err)
PanicOn(err)
if max != uint64(shard) {
panic(fmt.Sprintf("expected max (%v) to be == shard = %v", max, shard))
}
@ -217,12 +218,12 @@ func TestBolt_Max_on_many_containers(t *testing.T) {
// check for not found
max, err := tx.Max(index, field, view, uint64(200))
panicOn(err)
PanicOn(err)
if max != 0 {
panic("expected not found to give 0 max back with nil err")
}
max, err = tx.Max(index, field, view, uint64(400))
panicOn(err)
PanicOn(err)
if max != 0 {
panic("expected not found to give 0 max back with nil err")
}
@ -242,16 +243,16 @@ func TestBolt_SetBitmap(t *testing.T) {
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
PanicOn(err)
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
PanicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!!")
}
err = tx.Commit()
panicOn(err)
PanicOn(err)
//
// commited, so should be visible outside the txn
@ -259,13 +260,13 @@ func TestBolt_SetBitmap(t *testing.T) {
tx2, _ := dbwrap.NewTx(!writable, index, Txo{})
exists, err = tx2.Contains(index, field, view, shard, bitvalue)
panicOn(err)
PanicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!! on tx2")
}
n, err := tx2.Count(index, field, view, shard)
panicOn(err)
PanicOn(err)
if n != 1 {
panic(fmt.Sprintf("should have Count 1; instead n = %v", n))
}
@ -284,28 +285,28 @@ func TestBolt_OffsetRange(t *testing.T) {
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
PanicOn(err)
bitvalue2 := uint64(1<<20 + 1)
changed, err = tx.Add(index, field, view, shard, doBatched, bitvalue2)
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
PanicOn(err)
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
PanicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!!")
}
exists, err = tx.Contains(index, field, view, shard, bitvalue2)
panicOn(err)
PanicOn(err)
if !exists {
panic("ARG bitvalue2 was NOT SET!!!")
}
err = tx.Commit()
panicOn(err)
PanicOn(err)
offset := uint64(0 << 20)
start := uint64(0 << 16)
@ -313,7 +314,7 @@ func TestBolt_OffsetRange(t *testing.T) {
tx2, _ := dbwrap.NewTx(!writable, index, Txo{})
rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
PanicOn(err)
tx2.Rollback()
// should see our 1M value
@ -327,7 +328,7 @@ func TestBolt_OffsetRange(t *testing.T) {
offset = uint64(2 << 20)
tx3, _ := dbwrap.NewTx(!writable, index, Txo{})
rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx)
panicOn(err)
PanicOn(err)
tx3.Rollback()
//expect to see 3M == 3145728
@ -357,7 +358,7 @@ func TestBolt_Count_on_many_containers(t *testing.T) {
defer tx.Rollback()
n, err := tx.Count(index, field, view, shard)
panicOn(err)
PanicOn(err)
if int(n) != len(putmeValues) {
panic(fmt.Sprintf("expected Count of %v but got n=%v", len(putmeValues), n))
}
@ -374,7 +375,7 @@ func TestBolt_Count_dense_containers(t *testing.T) {
expected := 0
for i := uint64(0); i < (1<<16)+2; i += 2 {
changed, err := tx.Add(index, field, view, shard, doBatched, i)
panicOn(err)
PanicOn(err)
if changed <= 0 {
panic("wat? should have changed")
}
@ -383,7 +384,7 @@ func TestBolt_Count_dense_containers(t *testing.T) {
defer tx.Rollback()
n, err := tx.Count(index, field, view, shard)
panicOn(err)
PanicOn(err)
if int(n) != expected {
panic(fmt.Sprintf("expected Count of %v but got n=%v", expected, n))
}
@ -399,12 +400,12 @@ func TestBolt_ContainerIterator_on_empty(t *testing.T) {
defer tx.Rollback()
bitvalue := uint64(0)
citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue)
panicOn(err)
PanicOn(err)
defer citer.Close()
if found {
panic("should not have found anything")
}
panicOn(err)
PanicOn(err)
}
func TestBolt_ContainerIterator_on_one_bit(t *testing.T) {
@ -423,10 +424,10 @@ func TestBolt_ContainerIterator_on_one_bit(t *testing.T) {
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
PanicOn(err)
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
PanicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!!")
}
@ -437,7 +438,7 @@ func TestBolt_ContainerIterator_on_one_bit(t *testing.T) {
if !found {
panic("ContainerIterator did not find the 42 bit")
}
panicOn(err)
PanicOn(err)
defer citer.Close()
loopCount := 0
@ -481,10 +482,10 @@ func TestBolt_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) {
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
PanicOn(err)
exists, err := tx.Contains(index, field, view, shard, putme)
panicOn(err)
PanicOn(err)
if !exists {
panic("ARG putme was NOT SET!!!")
}
@ -517,7 +518,7 @@ func TestBolt_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) {
break
}
}
panicOn(err)
PanicOn(err)
}
func TestBolt_ContainerIterator_empty_iteration_loop(t *testing.T) {
@ -536,10 +537,10 @@ func TestBolt_ContainerIterator_empty_iteration_loop(t *testing.T) {
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
PanicOn(err)
exists, err := tx.Contains(index, field, view, shard, putme)
panicOn(err)
PanicOn(err)
if !exists {
panic("ARG putme was NOT SET!!!")
}
@ -547,7 +548,7 @@ func TestBolt_ContainerIterator_empty_iteration_loop(t *testing.T) {
// same Tx, continues in use.
citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(searchme))
panicOn(err)
PanicOn(err)
if found {
panic("ContainerIterator found the searchme, when it should not have")
}
@ -585,10 +586,10 @@ func TestBolt_ForEach_on_one_bit(t *testing.T) {
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
PanicOn(err)
exists, err := tx.Contains(index, field, view, shard, bitvalue)
panicOn(err)
PanicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!!")
}
@ -602,7 +603,7 @@ func TestBolt_ForEach_on_one_bit(t *testing.T) {
count += 1
return nil
})
panicOn(err)
PanicOn(err)
if count != 1 {
panic(fmt.Sprintf("Expected single iteration got %v ", count))
}
@ -637,7 +638,7 @@ func TestBolt_RemoveContainer_one_bit_test(t *testing.T) {
// delete, but rollback instead of commit
tx, _ := dbwrap.NewTx(writable, index, Txo{})
hi := highbits(putme)
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
PanicOn(tx.RemoveContainer(index, field, view, shard, hi))
tx.Rollback()
// verify that the rollback undid the deletion.
@ -648,15 +649,15 @@ func TestBolt_RemoveContainer_one_bit_test(t *testing.T) {
hi = highbits(putme)
exists, err := tx.Contains(index, field, view, shard, putme)
panicOn(err)
PanicOn(err)
if !exists {
panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme))
}
panicOn(tx.RemoveContainer(index, field, view, shard, hi))
PanicOn(tx.RemoveContainer(index, field, view, shard, hi))
exists, err = tx.Contains(index, field, view, shard, putme)
panicOn(err)
PanicOn(err)
if exists {
panic(fmt.Sprintf("ARG putme '%v' was SET even after RemoveContiner in this txn.", putme))
}
@ -700,7 +701,7 @@ func TestBolt_Remove_one_bit_test(t *testing.T) {
hi, lo := highbits(putme), lowbits(putme)
_, _ = hi, lo
_, err := tx.Remove(index, field, view, shard, hi)
panicOn(err)
PanicOn(err)
tx.Rollback()
// verify that the rollback undid the deletion.
@ -710,7 +711,7 @@ func TestBolt_Remove_one_bit_test(t *testing.T) {
tx, _ = dbwrap.NewTx(writable, index, Txo{})
exists, err := tx.Contains(index, field, view, shard, putme)
panicOn(err)
PanicOn(err)
if !exists {
panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme))
}
@ -718,7 +719,7 @@ func TestBolt_Remove_one_bit_test(t *testing.T) {
mustRemove(tx.Remove(index, field, view, shard, putme))
exists, err = tx.Contains(index, field, view, shard, putme)
panicOn(err)
PanicOn(err)
if exists {
panic(fmt.Sprintf("ARG putme '%v' was SET even after Remove in this txn.", putme))
}
@ -742,7 +743,7 @@ func TestBolt_Min_on_many_containers(t *testing.T) {
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
min, containersExist, err := tx.Min(index, field, view, shard)
_ = min
panicOn(err)
PanicOn(err)
if containersExist {
panic("no containers should exist")
}
@ -760,7 +761,7 @@ func TestBolt_Min_on_many_containers(t *testing.T) {
defer tx.Rollback()
min, containersExist, err = tx.Min(index, field, view, shard)
panicOn(err)
PanicOn(err)
if !containersExist {
panic("containers should exist")
}
@ -779,7 +780,7 @@ func TestBolt_CountRange_on_many_containers(t *testing.T) {
// verify no containers flag works
tx, _ := dbwrap.NewTx(!writable, index, Txo{})
n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
panicOn(err)
PanicOn(err)
if n != 0 {
panic("no containers should exist")
}
@ -797,7 +798,7 @@ func TestBolt_CountRange_on_many_containers(t *testing.T) {
defer tx.Rollback()
n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64)
panicOn(err)
PanicOn(err)
if n == 0 {
panic("containers should exist")
}
@ -826,7 +827,7 @@ func TestBolt_CountRange_middle_container(t *testing.T) {
// pick out just the middle container with the 1 bit set on it.
n, err := tx.CountRange(index, field, view, shard, 4, (2<<16)+1)
panicOn(err)
PanicOn(err)
if n != 1 {
panic("middle 1 bit container should exist")
}
@ -851,7 +852,7 @@ func TestBolt_CountRange_many_middle_container(t *testing.T) {
// get them all
n, err := tx.CountRange(index, field, view, shard, 0, (4<<16)+1)
panicOn(err)
PanicOn(err)
if n != 3 {
panic("count should have been all 3 bits")
}
@ -879,7 +880,7 @@ func TestBolt_UnionInPlace(t *testing.T) {
tx2, _ := dbwrap.NewTx(!writable, index, Txo{})
n, err := tx2.Count(index, field, view, shard)
panicOn(err)
PanicOn(err)
if n != 2 {
panic("should have 2 bits set")
}
@ -895,11 +896,11 @@ func TestBolt_UnionInPlace(t *testing.T) {
tx, _ := dbwrap.NewTx(writable, index, Txo{})
defer tx.Rollback()
err = tx.UnionInPlace(index, field, view, shard, others, others2, others3)
panicOn(err)
PanicOn(err)
// end game, check we got the union.
rbm, err := tx.RoaringBitmap(index, field, view, shard)
panicOn(err)
PanicOn(err)
n = rbm.Count()
if n != 7 {
panic("should have a total 3 + 3 +1 = 7 bits set on the containers")
@ -921,7 +922,7 @@ func TestBolt_RoaringBitmap(t *testing.T) {
defer tx.Rollback()
rbm, err := tx.RoaringBitmap(index, field, view, shard)
panicOn(err)
PanicOn(err)
slc := rbm.Slice()
if slc[0] != uint64(expected) {
@ -947,7 +948,7 @@ func TestBolt_ImportRoaringBits(t *testing.T) {
bits := []uint64{0, 2, 5, 1<<16 + 1, 2 << 16}
data := getTestBitmapAsRawRoaring(bits...)
itr, err := roaring.NewRoaringIterator(data)
panicOn(err)
PanicOn(err)
clear := false
logme := false
@ -956,11 +957,11 @@ func TestBolt_ImportRoaringBits(t *testing.T) {
if changed != len(bits) {
panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err))
}
panicOn(err)
PanicOn(err)
for _, v := range bits {
exists, err := tx.Contains(index, field, view, shard, v)
panicOn(err)
PanicOn(err)
if !exists {
panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v))
}
@ -973,11 +974,11 @@ func TestBolt_ImportRoaringBits(t *testing.T) {
if changed != 0 {
panic(fmt.Sprintf("should have not changed any bits on the second import, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err))
}
panicOn(err)
PanicOn(err)
for _, v := range bits {
exists, err := tx.Contains(index, field, view, shard, v)
panicOn(err)
PanicOn(err)
if !exists {
panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v))
}
@ -990,17 +991,17 @@ func TestBolt_ImportRoaringBits(t *testing.T) {
// clear 1 bit at a time
data := getTestBitmapAsRawRoaring(v)
itr, err := roaring.NewRoaringIterator(data)
panicOn(err)
PanicOn(err)
changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil)
_ = rowSet
if changed != 1 {
panic(fmt.Sprintf("should have changed 1 bit: '%v', rowSet='%#v', err='%v'", changed, rowSet, err))
}
panicOn(err)
PanicOn(err)
}
n, err := tx.Count(index, field, view, shard)
panicOn(err)
PanicOn(err)
if n != 0 {
panic(fmt.Sprintf("n = %v not zero so the clearbits didn't happen!", n))
}
@ -1027,12 +1028,12 @@ func TestBolt_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
bits := []uint64{0, 2, 1 << 16, 1<<16 + 2}
data := getTestBitmapAsRawRoaring(bits...)
itr, err := roaring.NewRoaringIterator(data)
panicOn(err)
PanicOn(err)
bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16}
data2 := getTestBitmapAsRawRoaring(bits2...)
itr2, err := roaring.NewRoaringIterator(data2)
panicOn(err)
PanicOn(err)
clear := false
logme := false
@ -1042,11 +1043,11 @@ func TestBolt_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
if changed != len(bits) {
panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err))
}
panicOn(err)
PanicOn(err)
for _, v := range bits {
exists, err := tx.Contains(index, field, view, shard, v)
panicOn(err)
PanicOn(err)
if !exists {
panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v))
}
@ -1059,7 +1060,7 @@ func TestBolt_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) {
if changed != 4 {
panic(fmt.Sprintf("should have changed 2 bits: the 1 and the 3, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err))
}
panicOn(err)
PanicOn(err)
}
func TestBolt_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
@ -1077,12 +1078,12 @@ func TestBolt_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} //, 5, 1<<16 + 1, 2 << 16}
data := getTestBitmapAsRawRoaring(bits...)
itr, err := roaring.NewRoaringIterator(data)
panicOn(err)
PanicOn(err)
bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16}
data2 := getTestBitmapAsRawRoaring(bits2...)
itr2, err := roaring.NewRoaringIterator(data2)
panicOn(err)
PanicOn(err)
clear := false
logme := false
@ -1092,11 +1093,11 @@ func TestBolt_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
if changed != len(bits) {
panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err))
}
panicOn(err)
PanicOn(err)
for _, v := range bits {
exists, err := tx.Contains(index, field, view, shard, v)
panicOn(err)
PanicOn(err)
if !exists {
panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v))
}
@ -1110,10 +1111,10 @@ func TestBolt_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) {
if changed != 2 {
panic(fmt.Sprintf("should have changed 1 bit: the 2, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err))
}
panicOn(err)
PanicOn(err)
n, err := tx.Count(index, field, view, shard)
panicOn(err)
PanicOn(err)
if n != 2 { // just the 0 and the 1<<16 bits should be left set.
panic(fmt.Sprintf("n = %v not 2 so the clearbits didn't happen!", n))
}
@ -1135,7 +1136,7 @@ func TestBolt_DeleteIndex(t *testing.T) {
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
PanicOn(err)
}
index2 := "i2" // should not be deleted, even though it shares a prefix with 'i'
@ -1143,38 +1144,38 @@ func TestBolt_DeleteIndex(t *testing.T) {
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
PanicOn(err)
for _, v := range bits {
exists, err := tx.Contains(index, field, view, shard, v)
panicOn(err)
PanicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!!")
}
}
exists, err := tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
PanicOn(err)
if !exists {
panic("ARG bitvalue was NOT SET!!! on index2")
}
err = tx.Commit()
panicOn(err)
PanicOn(err)
// end of setup
err = dbwrap.DeleteIndex(index)
panicOn(err)
PanicOn(err)
tx, _ = dbwrap.NewTx(!writable, index2, Txo{})
defer tx.Rollback()
exists, err = tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
PanicOn(err)
if !exists {
panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2))
}
for _, v := range bits {
exists, err = tx.Contains(index, field, view, shard, v)
panicOn(err)
PanicOn(err)
if exists {
allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false)
panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys))
@ -1199,9 +1200,9 @@ func TestBolt_DeleteIndex_over100k(t *testing.T) {
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
PanicOn(err)
if v%100000 == 0 {
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx, _ = dbwrap.NewTx(writable, index, Txo{})
}
}
@ -1211,25 +1212,25 @@ func TestBolt_DeleteIndex_over100k(t *testing.T) {
if changed <= 0 {
panic("should have changed")
}
panicOn(err)
PanicOn(err)
err = tx.Commit()
panicOn(err)
PanicOn(err)
// end of setup
err = dbwrap.DeleteIndex(index)
panicOn(err)
PanicOn(err)
tx, _ = dbwrap.NewTx(!writable, index2, Txo{})
defer tx.Rollback()
exists, err := tx.Contains(index2, field, view, shard, bitvalue)
panicOn(err)
PanicOn(err)
if !exists {
panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2))
}
for v := uint64(0); v < limit; v++ {
exists, err = tx.Contains(index, field, view, shard, v<<16)
panicOn(err)
PanicOn(err)
if exists {
allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false)
panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys))

90
catcher.go
View file

@ -20,13 +20,13 @@ import (
"github.com/pilosa/pilosa/v2/roaring"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
//txkey "github.com/pilosa/pilosa/v2/txkey"
. "github.com/pilosa/pilosa/v2/vprint"
)
// catcher is useful to report error locations with a
// stack dump before the complexity
// Stack dump before the complexity
// of the executor_test swallows up
// the location of a panic.
// the location of a PanicOn.
type catcherTx struct {
b Tx
}
@ -53,8 +53,8 @@ func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roar
func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
@ -67,8 +67,8 @@ func (c *catcherTx) Dump(short bool, shard uint64) {
func (c *catcherTx) Readonly() bool {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Readonly() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Readonly() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Readonly()
@ -81,8 +81,8 @@ func (tx *catcherTx) Pointer() string {
func (c *catcherTx) Rollback() {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
c.b.Rollback()
@ -92,8 +92,8 @@ func (c *catcherTx) Commit() error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Commit()
@ -103,8 +103,8 @@ func (c *catcherTx) RoaringBitmap(index, field, view string, shard uint64) (*roa
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.RoaringBitmap(index, field, view, shard)
@ -114,8 +114,8 @@ func (c *catcherTx) Container(index, field, view string, shard uint64, key uint6
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Container(index, field, view, shard, key)
@ -125,8 +125,8 @@ func (c *catcherTx) PutContainer(index, field, view string, shard uint64, key ui
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.PutContainer(index, field, view, shard, key, rc)
@ -136,8 +136,8 @@ func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.RemoveContainer(index, field, view, shard, key)
@ -155,8 +155,8 @@ func (c *catcherTx) Add(index, field, view string, shard uint64, batched bool, a
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Add() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Add() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Add(index, field, view, shard, batched, a...)
@ -166,8 +166,8 @@ func (c *catcherTx) Remove(index, field, view string, shard uint64, a ...uint64)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Remove(index, field, view, shard, a...)
@ -177,8 +177,8 @@ func (c *catcherTx) Contains(index, field, view string, shard uint64, key uint64
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Contains(index, field, view, shard, key)
@ -188,8 +188,8 @@ func (c *catcherTx) ContainerIterator(index, field, view string, shard uint64, f
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
@ -199,8 +199,8 @@ func (c *catcherTx) ForEach(index, field, view string, shard uint64, fn func(i u
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.ForEach(index, field, view, shard, fn)
@ -210,8 +210,8 @@ func (c *catcherTx) ForEachRange(index, field, view string, shard uint64, start,
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.ForEachRange(index, field, view, shard, start, end, fn)
@ -221,8 +221,8 @@ func (c *catcherTx) Count(index, field, view string, shard uint64) (uint64, erro
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Count(index, field, view, shard)
@ -232,8 +232,8 @@ func (c *catcherTx) Max(index, field, view string, shard uint64) (uint64, error)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Max(index, field, view, shard)
@ -243,8 +243,8 @@ func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool,
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Min(index, field, view, shard)
@ -254,8 +254,8 @@ func (c *catcherTx) UnionInPlace(index, field, view string, shard uint64, others
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.UnionInPlace(index, field, view, shard, others...)
@ -265,8 +265,8 @@ func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, e
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.CountRange(index, field, view, shard, start, end)
@ -276,8 +276,8 @@ func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start,
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see OffsetRange() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.OffsetRange(index, field, view, shard, offset, start, end)
@ -286,8 +286,8 @@ func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start,
func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)

14
client/client_it_test.go
View file

@ -1930,13 +1930,13 @@ func queryBalances(client *Client, acctOwnerID uint64, fieldAcct0, fieldAcct1 st
q := fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct0, acctOwnerID)
pql := NewPQLBaseQuery(q, indexAR, nil)
r, err := client.Query(pql)
panicOn(err)
PanicOn(err)
acct0bal = r.ResultList[0].(*ValCountResult).Val
q = fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct1, acctOwnerID)
pql = NewPQLBaseQuery(q, indexAR, nil)
r, err = client.Query(pql)
panicOn(err)
PanicOn(err)
acct1bal = r.ResultList[0].(*ValCountResult).Val
return
@ -1987,7 +1987,7 @@ func TestImportAtomicRecord(t *testing.T) {
}
data, err = proto.Marshal(ar)
panicOn(err)
PanicOn(err)
return
}
@ -2000,9 +2000,9 @@ func TestImportAtomicRecord(t *testing.T) {
expectedBalStartingAcct1 := int64(700)
data, err := setBal(expectedBalStartingAcct0, expectedBalStartingAcct1)
panicOn(err)
PanicOn(err)
err = client.importData(uri, "/import-atomic-record", data)
panicOn(err)
PanicOn(err)
// start the main test, reading two balances and writing two updates.
@ -2018,7 +2018,7 @@ func TestImportAtomicRecord(t *testing.T) {
}
data, err = setBal(expectedBalStartingAcct0-transferUSD, expectedBalStartingAcct1+transferUSD)
panicOn(err)
PanicOn(err)
//vv("sad path: transferUSD %v from %v -> %v, with power loss half-way through", transferUSD, fieldAcct0, fieldAcct1)
err = client.importData(uri, "/import-atomic-record?simPowerLossAfter=1", data)
@ -2045,7 +2045,7 @@ func TestImportAtomicRecord(t *testing.T) {
// happy path with no power failure half-way through.
err = client.importData(uri, "/import-atomic-record?simPowerLossAfter=0", data)
panicOn(err)
PanicOn(err)
endingBalanceAcct0, endingBalanceAcct1 = queryBalances(client, acctOwnerID, fieldAcct0, fieldAcct1)
// should have been applied this time.

13
cluster_internal_test.go
View file

@ -29,6 +29,7 @@ import (
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pilosa/pilosa/v2/topology"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
// GlobalPortMap avoids many races and port conflicts when setting
@ -137,7 +138,7 @@ func newHolderWithTempPath(tb testing.TB, backend string) *Holder {
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = backend
h := NewHolder(path, cfg)
panicOn(h.Open())
PanicOn(h.Open())
testhook.Cleanup(tb, func() {
h.Close()
})
@ -151,7 +152,7 @@ func newIndexWithTempPath(tb testing.TB, name string) *Index {
panic(err)
}
h := NewHolder(path, nil)
panicOn(h.Open())
PanicOn(h.Open())
index, err := h.CreateIndex(name, IndexOptions{})
testhook.Cleanup(tb, func() {
h.Close()
@ -232,7 +233,7 @@ func TestFragSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
shard = 1
tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
@ -241,7 +242,7 @@ func TestFragSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
shard = 2
tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
@ -251,7 +252,7 @@ func TestFragSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
shard = 3
tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
@ -261,7 +262,7 @@ func TestFragSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tests := []struct {
from *cluster

26
cmd/badloader/badloader.go
View file

@ -29,11 +29,9 @@ import (
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/http"
pnet "github.com/pilosa/pilosa/v2/net"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
//"log"
"os"
//"path/filepath"
//"sort"
"strconv"
"strings"
)
@ -62,13 +60,12 @@ func UploadTar(srcFile string, client *http.InternalClient) error {
lastIndex := ""
lastField := ""
lastShard := uint64(0)
//vv("top of tar loop")
n := 0
for {
header, err := tarReader.Next()
if err == io.EOF {
if header != nil {
panic("header should not be nil on err io.EOF")
PanicOn("header should not be nil on err io.EOF")
}
//submit any stuff we have left
if len(viewData) > 0 {
@ -76,18 +73,15 @@ func UploadTar(srcFile string, client *http.InternalClient) error {
Views: viewData,
}
// Submit(lastIndex, lastField, lastShard, request)
//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
uri := GetImportRoaringURI(lastIndex, lastShard)
err := client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request)
panicOn(err)
//vv("done with submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
PanicOn(err)
}
return nil
}
//vv("got header '%v'", header.Name)
n++
if n%500 == 0 {
vv("n = %v, progress, elapsed '%v'", n, time.Since(t0))
VV("n = %v, progress, elapsed '%v'", n, time.Since(t0))
}
parts := strings.Split(header.Name, "/")
//vv("parts = '%#v'", parts)
@ -106,7 +100,7 @@ func UploadTar(srcFile string, client *http.InternalClient) error {
}
//vv("about to submit lastIndex='%v' lastShard='%v'", lastIndex, lastShard)
uri := GetImportRoaringURI(lastIndex, lastShard)
panicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request))
PanicOn(client.ImportRoaring(context.Background(), uri, lastIndex, lastField, lastShard, false, request))
viewData = make(map[string][]byte)
//vv("done with submit lastIndex='%v' lastShard='%v'; took='%v'", lastIndex, lastShard, time.Since(t0))
@ -117,7 +111,7 @@ func UploadTar(srcFile string, client *http.InternalClient) error {
return err
}
if _, already := viewData[view]; already {
panic(fmt.Sprintf("view '%v' already present!", view))
PanicOn(fmt.Sprintf("view '%v' already present!", view))
}
viewData[view] = roaringData
lastIndex = index
@ -136,12 +130,12 @@ func main() {
host := "127.0.0.1:10101"
h := &gohttp.Client{}
c, err := http.NewInternalClient(host, h)
panicOn(err)
PanicOn(err)
tarSrcPath := "q2.tar.gz"
t0 := time.Now()
panicOn(UploadTar(tarSrcPath, c))
vv("total elapsed '%v'", time.Since(t0))
PanicOn(UploadTar(tarSrcPath, c))
VV("total elapsed '%v'", time.Since(t0))
}
var globURI *pnet.URI
@ -149,7 +143,7 @@ var globURI *pnet.URI
func init() {
var err error
globURI, err = pnet.NewURIFromHostPort("127.0.0.1", 10101)
panicOn(err)
PanicOn(err)
}
// get correct node to go to.

177
cmd/badloader/vprint.go
View file

@ -1,177 +0,0 @@
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main
import (
"fmt"
"io"
"os"
"path"
"runtime"
"runtime/debug"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("\n%s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}
// happy linter:
var _ = DirExists
var _ = FileExists
var _ = Caller
var _ = stack
var _ = RFC3339MsecTz0
var _ = RFC3339UsecTz0
var _ = AlwaysPrintf
var _ = FileSize

7
cmd/random-query/main.go
View file

@ -29,6 +29,7 @@ import (
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/http"
"github.com/pilosa/pilosa/v2/pql"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
// RandomQueryConfig
@ -210,7 +211,7 @@ NewSetup:
index := indexes[cfg.Rnd.Intn(len(indexes))]
pql, err := cfg.GenQuery(index)
panicOn(err)
PanicOn(err)
if cfg.Verbose {
fmt.Printf("pql = '%v'\n", pql)
@ -355,7 +356,7 @@ func (cfg *RandomQueryConfig) Setup(api API) (err error) {
pql := fmt.Sprintf("Rows(%v)", fld.Name)
res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql})
panicOn(err)
PanicOn(err)
if cfg.VeryVerbose {
fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0])
}
@ -411,7 +412,7 @@ func (cfg *RandomQueryConfig) AddIntField(index, field string, min, max pql.Deci
cfg.IndexMap[index] = f
}
if min.Scale != scale || max.Scale != scale {
panic(fmt.Sprintf("scale error; min scale %d, max scale %d, field scale %d, assumed they'd be equal",
PanicOn(fmt.Sprintf("scale error; min scale %d, max scale %d, field scale %d, assumed they'd be equal",
min.Scale, max.Scale, scale))
}

7
cmd/random-query/main_test.go
View file

@ -25,6 +25,7 @@ import (
"github.com/pilosa/pilosa/v2/http"
"github.com/pilosa/pilosa/v2/server"
"github.com/pilosa/pilosa/v2/test"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
func Test_RandomQuery(t *testing.T) {
@ -139,18 +140,18 @@ func Test_RandomQuery(t *testing.T) {
if err := nodes[0].API.Import(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
//qcx.Reset()
}
// end of setup.
panicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API)))
PanicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API)))
for j := 0; j < 4; j++ {
index := indexes[rand.Intn(len(indexes))]
pql, err := cfg.GenQuery(index)
panicOn(err)
PanicOn(err)
//vv("pql = '%v'", pql)

177
cmd/random-query/vprint.go
View file

@ -1,177 +0,0 @@
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main
import (
"fmt"
"io"
"os"
"path"
"runtime"
"runtime/debug"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("# %s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) int64 {
fi, err := os.Stat(name)
if err != nil {
return 0
}
return fi.Size()
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}
// happy linter:
var _ = DirExists
var _ = FileExists
var _ = Caller
var _ = stack
var _ = RFC3339MsecTz0
var _ = RFC3339UsecTz0
var _ = AlwaysPrintf
var _ = FileSize

27
cmd/slurp/slurp.go
View file

@ -33,6 +33,7 @@ import (
"github.com/pilosa/pilosa/v2"
"github.com/pilosa/pilosa/v2/http"
pnet "github.com/pilosa/pilosa/v2/net"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
// slurp: slurp is a load-tester for importing bulk data.
@ -59,7 +60,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
field := parts[2]
view := parts[4]
shard, err := strconv.ParseUint(parts[6], 10, 64)
panicOn(err)
PanicOn(err)
if index != r.lastIndex || field != r.lastField || shard != r.lastShard {
err := r.Upload()
if err != nil {
@ -83,7 +84,7 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
if err != nil {
return err
}
vv("Finished import %v", time.Since(r.start))
VV("Finished import %v", time.Since(r.start))
if r.profile != "" {
stopProfile(r.host, r.profile)
@ -103,21 +104,21 @@ func (r *stateMachine) NewHeader(h *tar.Header, tr *tar.Reader) error {
}
byteData, err := ioutil.ReadAll(tr)
panicOn(err)
PanicOn(err)
br := bytes.NewReader(byteData)
err = r.client.ImportFieldKeys(context.Background(), uri, index, fieldName, false, br)
if err != nil {
return err
}
default:
pilosa.VV("%v", h.Name)
VV("%v", h.Name)
index := parts[1]
partition, err := strconv.ParseUint(v, 10, 64)
if err != nil {
return err
}
byteData, err := ioutil.ReadAll(tr)
panicOn(err)
PanicOn(err)
br := bytes.NewReader(byteData)
err = r.client.ImportIndexKeys(context.Background(), uri, index, int(partition), false, br)
@ -175,10 +176,10 @@ func UploadTar(srcFile string, client *http.InternalClient, profile, host string
break
}
if err != nil {
panicOn(err)
PanicOn(err)
}
err = runner.NewHeader(header, tarReader)
panicOn(err)
PanicOn(err)
}
return nil
}
@ -193,7 +194,7 @@ func main() {
flag.Parse()
uri, err := pnet.NewURIFromAddress(host)
panicOn(err)
PanicOn(err)
globURI = uri
h := &gohttp.Client{}
@ -201,12 +202,12 @@ func main() {
startProfile(host)
}
c, err := http.NewInternalClient(host, h)
panicOn(err)
PanicOn(err)
t0 := time.Now()
println("uploading", tarSrcPath)
panicOn(UploadTar(tarSrcPath, c, profile, host))
vv("total elapsed '%v'", time.Since(t0))
PanicOn(UploadTar(tarSrcPath, c, profile, host))
VV("total elapsed '%v'", time.Since(t0))
}
func startProfile(host string) {
@ -247,10 +248,10 @@ func stopProfile(host, outfile string) {
}
fd, err := os.Create(outfile)
panicOn(err)
PanicOn(err)
defer fd.Close()
_, err = io.Copy(fd, resp.Body)
panicOn(err)
PanicOn(err)
}

177
cmd/slurp/vprint.go
View file

@ -1,177 +0,0 @@
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main
import (
"fmt"
"io"
"os"
"path"
"runtime"
"runtime/debug"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("\n%s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}
// happy linter:
var _ = DirExists
var _ = FileExists
var _ = Caller
var _ = stack
var _ = RFC3339MsecTz0
var _ = RFC3339UsecTz0
var _ = AlwaysPrintf
var _ = FileSize

42
dbshard.go
View file

@ -26,9 +26,9 @@ import (
rbfcfg "github.com/pilosa/pilosa/v2/rbf/cfg"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
"github.com/pilosa/pilosa/v2/storage"
//txkey "github.com/pilosa/pilosa/v2/txkey"
"github.com/pkg/errors"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
var _ = sort.Sort
@ -176,10 +176,10 @@ func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, er
}
}
if o.dbs != dbs {
panic(fmt.Sprintf("TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs))
PanicOn(fmt.Sprintf("TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs))
}
if o.Shard != dbs.Shard {
panic(fmt.Sprintf("shard disagreement! o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard)))
PanicOn(fmt.Sprintf("shard disagreement! o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard)))
}
var txns []Tx
@ -413,7 +413,7 @@ func (per *DBPerShard) LoadExistingDBs() (err error) {
func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string, holder *Holder) (d *DBPerShard) {
if holder.cfg == nil || holder.cfg.RBFConfig == nil || holder.cfg.StorageConfig == nil {
panic("must have holder.cfg.RBFConfig and holder.cfg.StorageConfig set here")
PanicOn("must have holder.cfg.RBFConfig and holder.cfg.StorageConfig set here")
}
useOpenList := 0
@ -522,7 +522,7 @@ func (dbs *DBShard) DumpAll() {
for i, ty := range dbs.types {
_ = i
tx, err := dbs.W[i].NewTx(!writable, "", Txo{Index: dbs.idx})
panicOn(err)
PanicOn(err)
defer tx.Rollback()
fmt.Printf("\n============= dumping dbs.W[%v] %v ========\n", i, ty)
tx.Dump(short, dbs.Shard)
@ -532,7 +532,7 @@ func (dbs *DBShard) DumpAll() {
case rbfTxn:
case boltTxn:
default:
panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
PanicOn(fmt.Sprintf("unknown txtyp: '%v'", ty))
}
}
fmt.Printf("\n============= end of DumpAll index='%v', shard=%v ========\n", dbs.Index, int(dbs.Shard))
@ -627,7 +627,7 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In
if len(per.types) == 1 && per.types[0] == roaringTxn {
// roaring txn are nil/fake anyway. Don't freak out.
} else {
panic(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.types[0]='%v'; len(per.types)=%v", dbs, per.types[0], len(per.types)))
PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.types[0]='%v'; len(per.types)=%v", dbs, per.types[0], len(per.types)))
}
}
if !ok {
@ -663,11 +663,11 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In
case boltTxn:
registry = globalBoltReg
default:
panic(fmt.Sprintf("unknown txtyp: '%v'", ty))
PanicOn(fmt.Sprintf("unknown txtyp: '%v'", ty))
}
path := dbs.pathForType(ty)
w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.StorageConfig)
panicOn(err)
PanicOn(err)
h := idx.Holder()
w.SetHolder(h)
dbs.Open = true
@ -688,7 +688,7 @@ func (per *DBPerShard) Del(dbs *DBShard) (err error) {
if err != nil {
return
}
panicOn(dbs.DeleteDBPath())
PanicOn(dbs.DeleteDBPath())
delete(per.Flatmap, flatkey{index: dbs.Index, shard: dbs.Shard})
// delete from the heirarchy
@ -703,7 +703,7 @@ func (per *DBPerShard) Close() (err error) {
for _, dbi := range per.dbh.Index {
for _, dbs := range dbi.Shard {
err = dbs.Close()
panicOn(err)
PanicOn(err)
}
}
return
@ -716,7 +716,7 @@ func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requir
n := len(f.types)
if n != 1 && n != 2 {
panic(fmt.Sprintf("internal error. only green or blue/green supported. we see types len %v", n))
PanicOn(fmt.Sprintf("internal error. only green or blue/green supported. we see types len %v", n))
}
var shards []map[uint64]bool
@ -810,7 +810,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r
ignoreEmpty := false
includeRoot := true
dbf, err := listDirUnderDir(path, includeRoot, ignoreEmpty)
panicOn(err)
PanicOn(err)
for _, nm := range dbf {
base := filepath.Base(nm)
@ -825,7 +825,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r
// Parse filename into integer.
shard, err := strconv.ParseUint(base[lenOfShardPrefix:], 10, 64)
if err != nil {
panicOn(err)
PanicOn(err)
continue
}
@ -938,7 +938,7 @@ func (dbs *DBShard) populateBlueFromGreen() (err error) {
n := len(dbs.W)
if n != 2 {
panic(fmt.Sprintf("populateBlueFromGreen did not find 2 open DBs: have %v", n))
PanicOn(fmt.Sprintf("populateBlueFromGreen did not find 2 open DBs: have %v", n))
}
dest := dbs.W[0] // blue
@ -948,11 +948,11 @@ func (dbs *DBShard) populateBlueFromGreen() (err error) {
// Since a shard is fairly small, we think one Tx will suffice.
readtx, err := src.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard})
panicOn(err)
PanicOn(err)
defer readtx.Rollback()
writetx, err := dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard})
panicOn(err)
PanicOn(err)
defer writetx.Rollback()
ctWriteCount := 0
@ -1024,18 +1024,18 @@ func (dbs *DBShard) verifyBlueEqualsGreen() (err error) {
n := len(dbs.W)
if n != 2 {
panic(fmt.Sprintf("verifyBlueEqualsGreen did not find 2 open DBs: have %v", n))
PanicOn(fmt.Sprintf("verifyBlueEqualsGreen did not find 2 open DBs: have %v", n))
}
blue := dbs.W[0]
green := dbs.W[1]
greentx, err := green.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard})
panicOn(err)
PanicOn(err)
defer greentx.Rollback()
bluetx, err := blue.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard})
panicOn(err)
PanicOn(err)
defer bluetx.Rollback()
for _, fld := range dbs.idx.Fields() {

47
dbshard_internal_test.go
View file

@ -25,6 +25,7 @@ import (
"github.com/pilosa/pilosa/v2/shardwidth"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
"github.com/pilosa/pilosa/v2/testhook"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
// Shard per db evaluation
@ -51,7 +52,7 @@ func TestShardPerDB_SetBit(t *testing.T) {
}
// commit the change, and verify it is still there
panicOn(tx.Commit())
PanicOn(tx.Commit())
// Close and reopen the fragment & verify the data.
err := f.Reopen() // roaring data not being flushed? red on roaring
@ -71,7 +72,7 @@ func TestShardPerDB_SetBit(t *testing.T) {
// test that we find all *local* shards
func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
tmpdir, err := testhook.TempDir(t, "Test_DBPerShard_GetShardsForIndex_LocalOnly")
panicOn(err)
PanicOn(err)
defer os.RemoveAll(tmpdir)
v2s := NewFieldView2Shards()
@ -92,13 +93,13 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
idx := makeSampleRoaringDir(t, tmpdir, index, src, 1, holder, v2s)
if idx == nil {
idx, err = NewIndex(holder, filepath.Join(tmpdir, index), index)
panicOn(err)
PanicOn(err)
}
estd := "rick/fields/_exists/views/standard"
std := "rick/fields/f/views/standard"
shards, err := holder.txf.GetShardsForIndex(idx, tmpdir+sep+std, false)
panicOn(err)
PanicOn(err)
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
if !shards[shard] {
@ -108,7 +109,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
if src == "roaring" {
// check estd too
shards, err = holder.txf.GetShardsForIndex(idx, tmpdir+sep+estd, false)
panicOn(err)
PanicOn(err)
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
if !shards[shard] {
panic(fmt.Sprintf("missing shard=%v from shards='%#v'", shard, shards))
@ -117,12 +118,12 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
// check GetSortedFieldViewList() and roaringGetFieldView2Shards()
vs, err := roaringGetFieldView2Shards(idx)
panicOn(err)
PanicOn(err)
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
fvs, err := tx.GetSortedFieldViewList(idx, shard)
panicOn(err)
PanicOn(err)
// expect these same two field/views for all 6 shards
expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
expect1 := txkey.FieldView{Field: "f", View: "standard"}
@ -149,7 +150,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
fvs, err := tx.GetSortedFieldViewList(idx, shard)
panicOn(err)
PanicOn(err)
// expect these same two field/views for all 6 shards
expect0 := txkey.FieldView{Field: "_exists", View: "standard"}
expect1 := txkey.FieldView{Field: "f", View: "standard"}
@ -242,12 +243,12 @@ func makeSampleRoaringDir(t *testing.T, root, index, backend string, minBytes in
}
path := root + sep + filepath.Dir(fn)
panicOn(os.MkdirAll(path, 0755))
PanicOn(os.MkdirAll(path, 0755))
fd, err := os.Create(root + sep + fn)
panicOn(err)
PanicOn(err)
if minBytes > 0 {
_, err := fd.Write(make([]byte, minBytes))
panicOn(err)
PanicOn(err)
}
fd.Close()
}
@ -256,10 +257,10 @@ func makeSampleRoaringDir(t *testing.T, root, index, backend string, minBytes in
func helperCreateDBShard(h *Holder, index string, shard uint64) *Index {
idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
panicOn(err)
PanicOn(err)
// TODO: It's not clear that this is actually doing anything.
dbs, err := h.txf.dbPerShard.GetDBShard(index, shard, idx)
panicOn(err)
PanicOn(err)
_ = dbs
return idx
}
@ -280,20 +281,20 @@ func makeRBFtestDB(path string, h *Holder, shard uint64) {
db := rbf.NewDB(path, nil)
err := db.Open()
panicOn(err)
PanicOn(err)
defer db.Close()
tx, err := db.Begin(true)
panicOn(err)
PanicOn(err)
err = tx.CreateBitmap("x")
panicOn(err)
PanicOn(err)
_, err = tx.Add("x", i)
panicOn(err)
PanicOn(err)
err = tx.Commit()
panicOn(err)
PanicOn(err)
}
func makeTxTestDBWithViewsShards(holder *Holder, idx *Index, exp *FieldView2Shards) {
@ -310,12 +311,12 @@ func makeTxTestDBWithViewsShards(holder *Holder, idx *Index, exp *FieldView2Shar
bits := []uint64{(shard << shardwidth.Exponent) + 1}
tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Shard: shard})
changeCount, err := tx.Add(idx.name, field, view, shard, batched, bits...)
panicOn(err)
PanicOn(err)
if changeCount != len(bits) {
panic(fmt.Sprintf("writing field '%v', view '%v' shard '%v', expected changeCount to equal len bits = %v but was %v", field, view, shard, len(bits), changeCount))
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
}
}
}
@ -325,7 +326,7 @@ func makeTxTestDBWithViewsShards(holder *Holder, idx *Index, exp *FieldView2Shar
// test that rbf can give us a map[view]*shardSet
func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) {
tmpdir, err := testhook.TempDir(t, "Test_DBPerShard_GetFieldView2Shards_map_from_RBF")
panicOn(err)
PanicOn(err)
defer os.RemoveAll(tmpdir)
cfg := mustHolderConfig()
@ -343,7 +344,7 @@ func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) {
}
idx, err := holder.createIndex(cim, false)
panicOn(err)
PanicOn(err)
exp := NewFieldView2Shards()
@ -360,7 +361,7 @@ func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) {
// setup is done
view2shard, err := holder.txf.GetFieldView2ShardsMapForIndex(idx)
panicOn(err)
PanicOn(err)
// compare against setup
if !view2shard.equals(exp) {

3
dbshard_test.go
View file

@ -25,6 +25,7 @@ import (
"github.com/pilosa/pilosa/v2/http"
"github.com/pilosa/pilosa/v2/server"
"github.com/pilosa/pilosa/v2/test"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
@ -94,7 +95,7 @@ func TestAPI_SimplerOneNode_ImportColumnKey(t *testing.T) {
if err := m0.API.Import(ctx, qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
//select {}

17
executor_test.go
View file

@ -46,6 +46,7 @@ import (
"github.com/pilosa/pilosa/v2/storage"
"github.com/pilosa/pilosa/v2/test"
"github.com/pilosa/pilosa/v2/testhook"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
)
@ -3166,12 +3167,12 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `
Set(0, fint=1)
Set(1, fint=2)
Set(2,fint=-2)
Set(3,fint=-1)
Set(4,fint=4)
Set(10, fint=0)
Set(100, fint=0)
Set(1000, fint=0)
@ -4063,12 +4064,12 @@ func TestExecutor_Execute_All(t *testing.T) {
if err := m0.API.Import(context.Background(), qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
i0, err := m0.API.Index(context.Background(), "i")
panicOn(err)
PanicOn(err)
if i0 == nil {
panic("nil index i0?")
PanicOn("nil index i0?")
}
tests := []struct {
@ -4145,7 +4146,7 @@ func TestExecutor_Execute_All(t *testing.T) {
if err := c.GetNode(0).API.Import(context.Background(), qcx, req); err != nil {
t.Fatal(err)
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
tests := []struct {
qry string
@ -4676,7 +4677,7 @@ func benchmarkExistence(nn bool, b *testing.B) {
if err := nodeAPI.Import(context.Background(), qcx, req); err != nil {
b.Fatal(err)
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
}
}

17
field_internal_test.go
View file

@ -29,6 +29,7 @@ import (
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/testhook"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
// Ensure a bsiGroup can adjust to its baseValue.
@ -212,7 +213,7 @@ func NewTestField(t *testing.T, opts FieldOption) *TestField {
cfg := DefaultHolderConfig()
cfg.StorageConfig.Backend = CurrentBackendOrDefault()
h := NewHolder(path, cfg)
panicOn(h.Open())
PanicOn(h.Open())
idx, err := h.CreateIndex("i", IndexOptions{})
if err != nil {
@ -238,7 +239,7 @@ func OpenField(t *testing.T, opts FieldOption) *TestField {
// Close closes the field and removes the underlying data.
func (f *TestField) Close() error {
if f.idx != nil {
panicOn(f.idx.holder.txf.CloseIndex(f.idx))
PanicOn(f.idx.holder.txf.CloseIndex(f.idx))
}
defer os.RemoveAll(f.Path())
return f.Field.Close()
@ -335,7 +336,7 @@ func TestField_RowTime(t *testing.T) {
f.MustSetBit(tx, 1, 4, time.Date(2010, time.January, 6, 12, 0, 0, 0, time.UTC))
f.MustSetBit(tx, 1, 5, time.Date(2010, time.January, 5, 13, 0, 0, 0, time.UTC))
panicOn(tx.Commit())
PanicOn(tx.Commit())
// obtain 2nd transaction to read it back.
tx = f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Field: f.Field, Shard: 0})
@ -610,14 +611,14 @@ func TestBSIGroup_importValue(t *testing.T) {
if err := f.importValue(qcx, tt.columnIDs, tt.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
qcx.Reset()
if row, err := f.Range(qcx, f.name, pql.EQ, tt.checkVal); err != nil {
t.Fatalf("test %d, getting range: %s", i, err.Error())
} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
qcx.Reset()
} // loop
}
@ -682,7 +683,7 @@ func TestIntField_MinMaxForShard(t *testing.T) {
if err := f.importValue(qcx, test.columnIDs, test.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
qcx.Reset()
shard := uint64(0)
@ -909,7 +910,7 @@ func TestBSIGroup_TxReopenDB(t *testing.T) {
if err := f.importValue(qcx, tt.columnIDs, tt.values, options); err != nil {
t.Fatalf("test %d, importing values: %s", i, err.Error())
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
qcx.Reset()
if row, err := f.Range(qcx, f.name, pql.EQ, tt.checkVal); err != nil {
@ -917,7 +918,7 @@ func TestBSIGroup_TxReopenDB(t *testing.T) {
} else if !reflect.DeepEqual(row.Columns(), tt.expCols) {
t.Fatalf("test %d, expected columns: %v, but got: %v", i, tt.expCols, row.Columns())
}
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
qcx.Reset()
} // loop

15
fragment.go
View file

@ -50,6 +50,7 @@ import (
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pilosa/pilosa/v2/topology"
"github.com/pilosa/pilosa/v2/tracing"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
)
@ -212,7 +213,7 @@ func newFragment(holder *Holder, spec fragSpec, shard uint64, flags byte) *fragm
idx := holder.Index(spec.index.name)
if idx == nil {
panic(fmt.Sprintf("got nil idx back for '%v' from holder!", spec.index))
PanicOn(fmt.Sprintf("got nil idx back for '%v' from holder!", spec.index))
}
f := &fragment{
@ -616,7 +617,7 @@ func (f *fragment) row(tx Tx, rowID uint64) (*Row, error) {
func (f *fragment) mustRow(tx Tx, rowID uint64) *Row {
row, err := f.row(tx, rowID)
if err != nil {
panic(err)
PanicOn(err)
}
return row
}
@ -1073,7 +1074,7 @@ func (f *fragment) setValueBase(txOrig Tx, columnID uint64, bitDepth uint64, val
tx = f.idx.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Fragment: f, Shard: f.shard})
defer func() {
if err == nil {
panicOn(tx.Commit())
PanicOn(tx.Commit())
} else {
tx.Rollback()
}
@ -2070,7 +2071,9 @@ func (f *fragment) Blocks() ([]FragmentBlock, error) {
idx := f.holder.Index(f.index())
if idx == nil {
panic(fmt.Sprintf("index() was nil in fragment.Blocks(): f.index()='%v'\n", f.index()))
err := fmt.Errorf("index() was nil in fragment.Blocks(): f.index()='%v'", f.index())
PanicOn(err)
return nil, err
}
tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -2404,7 +2407,7 @@ func (p *parallelSlices) fullPrune() {
return
}
if len(p.rows) != len(p.cols) {
panic("parallelSlices must have same length for rows and columns")
PanicOn("parallelSlices must have same length for rows and columns")
}
unsorted := p.prune()
if unsorted {
@ -2901,7 +2904,7 @@ func (f *fragment) snapshot() (err error) {
f.path(), mappedIn, mappedOut, unmappedIn, errs)
}
} else {
err = fmt.Errorf("non-error panic: %v", r)
err = fmt.Errorf("non-error PanicOn: %v", r)
}
}
}()

127
fragment_internal_test.go
View file

@ -40,6 +40,7 @@ import (
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/storage"
"github.com/pilosa/pilosa/v2/testhook"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
@ -76,7 +77,7 @@ func TestFragment_SetBit(t *testing.T) {
}
// commit the change, and verify it is still there
panicOn(tx.Commit())
PanicOn(tx.Commit())
// Close and reopen the fragment & verify the data.
err := f.Reopen()
@ -114,7 +115,7 @@ func TestFragment_ClearBit(t *testing.T) {
}
// The Reopen below implies this test is looking at storage consistency.
// In that spirit, we will check that the Tx Commit is visible afterwards.
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -200,7 +201,7 @@ func TestFragment_ClearRow(t *testing.T) {
if n := f.mustRow(tx, 1000).Count(); n != 0 {
t.Fatalf("unexpected count: %d", n)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -247,7 +248,7 @@ func TestFragment_SetRow(t *testing.T) {
t.Fatalf("expected changed value: %v", changed)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -260,7 +261,7 @@ func TestFragment_SetRow(t *testing.T) {
t.Fatalf("unexpected count after set row: %d", n)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
// Close and reopen the fragment & verify the data.
@ -285,7 +286,7 @@ func TestFragment_SetRow(t *testing.T) {
if cols := f.mustRow(tx, rowID).Columns(); len(cols) != 0 {
t.Fatalf("expected setting a row with no entries to clear the cache")
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
}
// Ensure a fragment can set & read a value.
@ -543,7 +544,7 @@ func TestFragment_Sum(t *testing.T) {
}
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -576,7 +577,7 @@ func TestFragment_Sum(t *testing.T) {
t.Fatal(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -615,7 +616,7 @@ func TestFragment_MinMax(t *testing.T) {
t.Fatal(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
// the new tx is shared by Min/Max below.
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
@ -1215,7 +1216,7 @@ func TestFragment_Snapshot(t *testing.T) {
} else if _, err := f.clearBit(tx, 1000, 1); err != nil {
t.Fatal(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -1308,7 +1309,7 @@ func TestFragment_Top_Filter(t *testing.T) {
t.Fatalf("setAttrs: %v", err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -1481,7 +1482,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
f := frag
if err := f.Open(); err != nil {
panic(err)
PanicOn(err)
}
defer f.Clean(t)
@ -1538,7 +1539,7 @@ func TestFragment_Checksum(t *testing.T) {
} else if _, err := f.setBit(tx, HashBlockSize*2, 200); err != nil {
t.Fatal(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
// Ensure new checksum is different.
if chksum, err := f.Checksum(); err != nil {
@ -1561,7 +1562,7 @@ func TestFragment_Blocks(t *testing.T) {
if _, err := f.setBit(tx, 0, 0); err != nil {
t.Fatal(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
blocks, err := f.Blocks() // FAIL: TestFragment_Blocks b/c 0 blocks back
if err != nil {
t.Fatal(err)
@ -1575,7 +1576,7 @@ func TestFragment_Blocks(t *testing.T) {
if _, err := f.setBit(tx, 20, 0); err != nil {
t.Fatal(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
blocks, err = f.Blocks()
if err != nil {
t.Fatal(err)
@ -1590,7 +1591,7 @@ func TestFragment_Blocks(t *testing.T) {
if _, err := f.setBit(tx, 20, 100); err != nil {
t.Fatal(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
blocks, err = f.Blocks()
if err != nil {
t.Fatal(err)
@ -1609,7 +1610,7 @@ func TestFragment_Blocks_Empty(t *testing.T) {
if _, err := f.setBit(tx, 100, 1); err != nil {
t.Fatal(err)
}
panicOn(tx.Commit()) // f.Blocks() will start a new Tx, so the SetBit needs to be visible before that.
PanicOn(tx.Commit()) // f.Blocks() will start a new Tx, so the SetBit needs to be visible before that.
// Ensure checksum for block 1 is blank.
if blocks, err := f.Blocks(); err != nil {
@ -1641,7 +1642,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
t.Fatalf("unexpected cache len: %d", cache.Len())
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
// Reopen the fragment.
if err := f.Reopen(); err != nil {
@ -1692,7 +1693,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
}
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = index.holder.txf.NewTx(Txo{Write: !writable, Index: index, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -1837,7 +1838,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
}
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -2178,7 +2179,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) {
t.Fatalf("bulk importing ids: %v", err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -2200,7 +2201,7 @@ func TestFragment_ImportSet_WithTxCommit(t *testing.T) {
t.Fatalf("bulk clearing ids: %v", err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -2230,12 +2231,12 @@ func TestFragment_ConcurrentImport(t *testing.T) {
eg := errgroup.Group{}
eg.Go(func() error {
tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
defer func() { panicOn(tx.Commit()) }()
defer func() { PanicOn(tx.Commit()) }()
return f.bulkImportStandard(tx, []uint64{1, 2}, []uint64{1, 2}, &ImportOptions{})
})
eg.Go(func() error {
tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
defer func() { panicOn(tx.Commit()) }()
defer func() { PanicOn(tx.Commit()) }()
return f.bulkImportStandard(tx, []uint64{3, 4}, []uint64{3, 4}, &ImportOptions{})
})
err := eg.Wait()
@ -2457,7 +2458,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) {
t.Fatalf("bulk importing ids: %v", err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -2479,7 +2480,7 @@ func TestFragment_ImportMutex_WithTxCommit(t *testing.T) {
t.Fatalf("bulk clearing ids: %v", err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -2707,7 +2708,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) {
t.Fatalf("bulk importing ids: %v", err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -2729,7 +2730,7 @@ func TestFragment_ImportBool_WithTxCommit(t *testing.T) {
t.Fatalf("bulk importing ids: %v", err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -3059,7 +3060,7 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
// is excessive. force storage into snapshotted state, then use import
// to generate an op log and/or snapshot.
itr, err := roaring.NewRoaringIterator(data)
panicOn(err)
PanicOn(err)
_, _, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, false, false, 0, nil)
if err != nil {
b.Errorf("import error: %v", err)
@ -3143,15 +3144,15 @@ func initBigFrag(tb testing.TB) {
data := getZipfRowsSliceRoaring(10000000, i, 0, ShardWidth)
err := f.importRoaringT(tx, data, false)
if err != nil {
panic(fmt.Sprintf("setting up fragment data: %v", err))
PanicOn(fmt.Sprintf("setting up fragment data: %v", err))
}
}
err := f.Close()
if err != nil {
panic(fmt.Sprintf("closing fragment: %v", err))
PanicOn(fmt.Sprintf("closing fragment: %v", err))
}
bigFrag = f.path()
panicOn(tx.Commit())
PanicOn(tx.Commit())
}
}
@ -3178,9 +3179,9 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) {
fi.Close()
h := NewHolder(fi.Name(), nil)
panicOn(h.Open())
PanicOn(h.Open())
idx, err := h.CreateIndex("i", IndexOptions{})
panicOn(err)
PanicOn(err)
f := newFragment(h, makeTestFragSpec(fi.Name(), "i", "f", viewStandard), 0, 0)
err = f.Open()
@ -3200,7 +3201,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) {
if err != nil {
b.Fatalf("bulkImport: %v", err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
f.Clean(b)
h.Close()
}
@ -3375,7 +3376,7 @@ func getZipfRowsSliceRoaring(numRows uint64, seed int64, startCol, endCol uint64
buf := bytes.NewBuffer(make([]byte, 0, 100000))
_, err := b.WriteTo(buf)
if err != nil {
panic(err)
PanicOn(err)
}
return buf.Bytes()
}
@ -3401,7 +3402,7 @@ func getUniformRowsSliceRoaring(numRows uint64, seed int64, startCol, endCol uin
buf := bytes.NewBuffer(make([]byte, 0, 100000))
_, err := b.WriteTo(buf)
if err != nil {
panic(err)
PanicOn(err)
}
return buf.Bytes()
}
@ -3428,7 +3429,7 @@ func getUpdataRoaring(numRows, numCols uint64, seed int64) []byte {
buf := bytes.NewBuffer(make([]byte, 0, 100000))
_, err := b.WriteTo(buf)
if err != nil {
panic(err)
PanicOn(err)
}
return buf.Bytes()
}
@ -3582,7 +3583,7 @@ func mustOpenBSIFragment(tb testing.TB, index, field, view string, shard uint64)
func newTestHolder(tb testing.TB) *Holder {
path, _ := testhook.TempDirInDir(tb, *TempDir, "holder-dir")
h := NewHolder(path, mustHolderConfig())
panicOn(h.Open())
PanicOn(h.Open())
testhook.Cleanup(tb, func() {
h.Close()
})
@ -3601,13 +3602,13 @@ func fragTestMustOpenIndex(index string, holder *Holder, opt IndexOptions) *Inde
holder.mu.Lock()
idx, err := holder.createIndex(cim, false)
holder.mu.Unlock()
panicOn(err)
PanicOn(err)
idx.keys = opt.Keys
idx.trackExistence = opt.TrackExistence
if err := idx.Open(); err != nil {
panic(err)
PanicOn(err)
}
return idx
}
@ -3625,14 +3626,14 @@ func mustOpenFragmentFlags(tb testing.TB, index, field, view string, shard uint6
}
fragDir := fmt.Sprintf("%v/%v/views/%v/fragments/", idx.path, field, view)
panicOn(os.MkdirAll(fragDir, 0777))
PanicOn(os.MkdirAll(fragDir, 0777))
fragPath := fragDir + fmt.Sprintf("%v", shard)
f := newFragment(th, makeTestFragSpec(fragPath, index, field, view), shard, flags)
tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: shard})
testhook.Cleanup(tb, func() {
tx.Rollback()
panicOn(idx.holder.txf.CloseIndex(idx))
PanicOn(idx.holder.txf.CloseIndex(idx))
})
f.CacheType = cacheType
@ -3641,7 +3642,7 @@ func mustOpenFragmentFlags(tb testing.TB, index, field, view string, shard uint6
}
if err := f.Open(); err != nil {
panic(err)
PanicOn(err)
}
return f, idx, tx
}
@ -3676,7 +3677,7 @@ func (f *fragment) Reopen() error {
func (f *fragment) mustSetBits(tx Tx, rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.setBit(tx, rowID, columnID); err != nil {
panic(err)
PanicOn(err)
}
}
}
@ -3737,7 +3738,7 @@ func TestFragment_RowsIteration(t *testing.T) {
} else if _, err := f.setBit(tx, 2, 166000); err != nil {
t.Fatal(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -3928,7 +3929,7 @@ func toRowsCols(roaring []uint64) (rowIDs, colIDs []uint64) {
func calcTop(rowIDs, colIDs []uint64) []Pair {
if len(rowIDs) != len(colIDs) {
panic("row and col ids must be of equal len")
PanicOn("row and col ids must be of equal len")
}
// make map of rowID to colID set in order to dedup
counts := make(map[uint64]map[uint64]struct{})
@ -4166,7 +4167,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
f.mustSetBits(tx, 2, 0)
f.mustSetBits(tx, 3, 0)
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -4214,7 +4215,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
f.mustSetBits(tx, 5, 0)
f.mustSetBits(tx, 7, 0)
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -4262,7 +4263,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
f.mustSetBits(tx, 2, 0)
f.mustSetBits(tx, 3, 0)
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -4299,7 +4300,7 @@ func TestFragmentRowIterator_WithTxCommit(t *testing.T) {
f.mustSetBits(tx, 5, 0)
f.mustSetBits(tx, 7, 0)
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -4535,7 +4536,7 @@ func TestFragmentBSIUnsigned(t *testing.T) {
t.Fatalf("no change when setting col %d to %d", uint64(i), int64(i))
}
}
panicOn(tx.Commit()) // t.Run beolow on different goro and so need their own Tx anyway.
PanicOn(tx.Commit()) // t.Run beolow on different goro and so need their own Tx anyway.
// Generate a list of columns.
cols := make([]uint64, 1<<k)
@ -4719,7 +4720,7 @@ func TestFragmentBSIUnsigned_WithTxCommit(t *testing.T) {
}
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -4882,7 +4883,7 @@ func TestFragmentBSISigned(t *testing.T) {
}
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
@ -5094,7 +5095,7 @@ func TestImportClearRestart(t *testing.T) {
if err != nil {
t.Fatalf("closing fragment: %v", err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
err = f.Open()
tx = idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
@ -5114,14 +5115,14 @@ func TestImportClearRestart(t *testing.T) {
h := newTestHolder(t)
idx2, err := h.CreateIndex("i", IndexOptions{})
_ = idx2
panicOn(err)
PanicOn(err)
// OVERWRITING the f.path with a new fragment
f2 := newFragment(h, makeTestFragSpec(f.path(), "i", "f", viewStandard), 0, 0)
f2.MaxOpN = maxOpN
f2.CacheType = f.CacheType
panicOn(tx.Commit()) // match the f.closeStorage which overlaps the f2 creation.
PanicOn(tx.Commit()) // match the f.closeStorage which overlaps the f2 creation.
tx2 := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f2, Shard: f2.shard})
defer tx2.Rollback()
@ -5158,7 +5159,7 @@ func TestImportClearRestart(t *testing.T) {
check(t, tx2, f2, exp)
panicOn(tx2.Commit())
PanicOn(tx2.Commit())
h3 := NewHolder(filepath.Dir(f2.path()), nil)
testhook.Cleanup(t, func() {
@ -5167,7 +5168,7 @@ func TestImportClearRestart(t *testing.T) {
idx3, err := h3.CreateIndex("i", IndexOptions{})
_ = idx3
panicOn(err)
PanicOn(err)
f3 := newFragment(h3, makeTestFragSpec(f2.path(), "i", "f", viewStandard), 0, 0)
f3.MaxOpN = maxOpN
@ -5291,7 +5292,7 @@ func TestImportMultipleValues(t *testing.T) {
}
// probably too slow, would hit disk alot:
//panicOn(tx.Commit())
//PanicOn(tx.Commit())
//tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard:f.shard, ShardSet:true})
//defer tx.Rollback()
@ -5398,7 +5399,7 @@ func TestFragmentConcurrentReadWrite(t *testing.T) {
return errors.Wrap(err, "setting bit")
}
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
return nil
})
@ -5423,7 +5424,7 @@ func TestRemapCache(t *testing.T) {
defer f.Close()
index, field, view, shard := f.index(), f.field(), f.view(), f.shard
// request a panic that doesn't kill the program on fault
// request a PanicOn that doesn't kill the program on fault
wouldFault := debug.SetPanicOnFault(true)
defer func() {
debug.SetPanicOnFault(wouldFault)
@ -5435,7 +5436,7 @@ func TestRemapCache(t *testing.T) {
t.Fatalf("segfault trapped during remap test (expected failure mode)")
}
}
t.Fatalf("unexpected panic: %v", r)
t.Fatalf("unexpected PanicOn: %v", r)
}
}()

3
holder.go
View file

@ -37,6 +37,7 @@ import (
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pilosa/pilosa/v2/topology"
"github.com/pilosa/pilosa/v2/tracing"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
@ -306,7 +307,7 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
storage.SetRowCacheOn(cfg.RowcacheOn)
txf, err := NewTxFactory(cfg.StorageConfig.Backend, h.IndexesPath(), h)
panicOn(err)
PanicOn(err)
h.txf = txf
h.txf.blueGreenOffIfRunningBlueGreen()

3
holder_internal_test.go
View file

@ -22,6 +22,7 @@ import (
"github.com/pilosa/pilosa/v2/disco"
"github.com/pilosa/pilosa/v2/testhook"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
var _ = fmt.Printf
@ -113,7 +114,7 @@ func testMustHaveBit(t *testing.T, h *Holder, index, field string, rowID, column
// hmm... if its a new holder, meta data isn't there, so ask for it.
idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
panicOn(err)
PanicOn(err)
f := idx.Field(field)
if f == nil {

1
index.go
View file

@ -28,6 +28,7 @@ import (
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
"github.com/pilosa/pilosa/v2/testhook"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)

5
pilosa_internal_test.go
View file

@ -20,6 +20,7 @@ import (
"testing"
"github.com/pilosa/pilosa/v2/roaring"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
func TestValidateName(t *testing.T) {
@ -76,7 +77,7 @@ func TestAPI_CombineForExistence(t *testing.T) {
bm := roaring.NewBitmap(pos(1, 1), pos(1, 2), pos(1, 3), pos(1, 65537), pos(1, 65538), pos(2, 1), pos(2, 2), pos(2, 5), pos(2, 65537), pos(2, 65538))
buf := new(bytes.Buffer)
_, err := bm.WriteTo(buf)
panicOn(err)
PanicOn(err)
raw := buf.Bytes()
results, err := combineForExistence(raw)
if err != nil {
@ -84,7 +85,7 @@ func TestAPI_CombineForExistence(t *testing.T) {
}
bm2 := roaring.NewBitmap()
_, _, err = bm2.ImportRoaringBits(results, false, false, 1<<shardVsContainerExponent)
panicOn(err)
PanicOn(err)
expected := []uint64{1, 2, 3, 5, 65537, 65538}
got := bm2.Slice()
if !reflect.DeepEqual(got, expected) {

13
pprof.go
View file

@ -24,6 +24,7 @@ import (
_ "net/http/pprof" // Imported for its side-effect of registering pprof endpoints with the server.
"github.com/pilosa/pilosa/v2/storage"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
// CPUProfileForDur (where "Dur" is short for "Duration"), is used for
@ -37,7 +38,7 @@ func CPUProfileForDur(dur time.Duration, outpath string) {
}
path := outpath + "." + backend
f, err := os.Create(path)
panicOn(err)
PanicOn(err)
if dur == 0 {
dur = time.Minute
@ -63,7 +64,7 @@ func MemProfileForDur(dur time.Duration, outpath string) {
}
path := outpath + "." + backend
f, err := os.Create(path)
panicOn(err)
PanicOn(err)
if dur == 0 {
dur = time.Minute
@ -73,7 +74,7 @@ func MemProfileForDur(dur time.Duration, outpath string) {
<-time.After(dur)
runtime.GC() // get up-to-date statistics
if err := pprof.WriteHeapProfile(f); err != nil {
panic(fmt.Sprintf("could not write memory profile: %v", err))
PanicOn(fmt.Sprintf("could not write memory profile: %v", err))
}
f.Close()
AlwaysPrintf("wrote memory profile after dur '%v', output: '%v'", dur, path)
@ -91,7 +92,7 @@ var _ = pprofProfile{}
func newPprof() (pp *pprofProfile) {
pp = &pprofProfile{}
f, err := os.Create("cpu.manual.pprof")
panicOn(err)
PanicOn(err)
pp.fdCpu = f
_ = pprof.StartCPUProfile(pp.fdCpu)
@ -104,11 +105,11 @@ func (pp *pprofProfile) Close() {
pp.fdCpu.Close()
f, err := os.Create("mem.manual.pprof")
panicOn(err)
PanicOn(err)
runtime.GC() // get up-to-date statistics
if err := pprof.WriteHeapProfile(f); err != nil {
panic(fmt.Sprintf("could not write memory profile: %v", err))
PanicOn(fmt.Sprintf("could not write memory profile: %v", err))
}
f.Close()
}

4
rbf.go
View file

@ -32,7 +32,7 @@ import (
txkey "github.com/pilosa/pilosa/v2/short_txkey"
"github.com/pilosa/pilosa/v2/storage"
//txkey "github.com/pilosa/pilosa/v2/txkey"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
)
@ -402,7 +402,7 @@ func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fr
func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
b, err := tx.RoaringBitmap(index, field, view, shard)
panicOn(err)
PanicOn(err)
return b.Iterator()
}

59
rbf/cursor_internal_test.go
View file

@ -20,6 +20,7 @@ import (
"testing"
"github.com/pilosa/pilosa/v2/roaring"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
func getRoaringIter(bitsToSet ...uint64) roaring.RoaringIterator {
@ -28,15 +29,15 @@ func getRoaringIter(bitsToSet ...uint64) roaring.RoaringIterator {
changed := b.DirectAddN(bitsToSet...)
n := len(bitsToSet)
if changed != n {
panic(fmt.Sprintf("changed=%v but bitsToSet len = %v", changed, n))
PanicOn(fmt.Sprintf("changed=%v but bitsToSet len = %v", changed, n))
}
buf := bytes.NewBuffer(make([]byte, 0, 100000))
_, err := b.WriteTo(buf)
if err != nil {
panic(err)
PanicOn(err)
}
itr, err := roaring.NewRoaringIterator(buf.Bytes())
panicOn(err)
PanicOn(err)
return itr
}
@ -59,14 +60,14 @@ func TestCursor_RoaringImport(t *testing.T) {
defer tx.Rollback()
changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
panicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
_ = rowSet
if changed != 1 {
t.Fatalf("expected 1 changed, got %v", changed)
}
if false {
cur, err := tx.cursor(name)
panicOn(err)
PanicOn(err)
cur.dump()
@ -93,7 +94,7 @@ func TestCursor_RoaringImport_clear_bits(t *testing.T) {
defer tx.Rollback()
changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
panicOn(err)
PanicOn(err)
_ = rowSet
if changed != 1 {
t.Fatalf("expected 1 changed, got %v", changed)
@ -104,7 +105,7 @@ func TestCursor_RoaringImport_clear_bits(t *testing.T) {
itr2 := getRoaringIter([]uint64{1}...)
changed, rowSet, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil)
panicOn(err)
PanicOn(err)
_ = rowSet
if changed != 1 {
t.Fatalf("expected 1 changed on clear true, got %v", changed)
@ -112,7 +113,7 @@ func TestCursor_RoaringImport_clear_bits(t *testing.T) {
if false {
cur, err := tx.cursor(name)
panicOn(err)
PanicOn(err)
cur.dump()
@ -152,14 +153,14 @@ func TestCursor_RoaringImport_two_leaves(t *testing.T) {
defer tx.Rollback()
changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
panicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
_ = rowSet
if changed != 6000 {
t.Fatalf("expected 6000 bits changed, got %v", changed)
}
if false {
cur, err := tx.cursor(name)
panicOn(err)
PanicOn(err)
cur.dump()
@ -211,7 +212,7 @@ func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) {
//vv("DONE WITH Add()")
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
panicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != expectedBitsChanged-3000 {
t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged-3000, changed)
}
@ -219,7 +220,7 @@ func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) {
//vv("about to do itr2, that starts with key %v", itr2.ContainerKeys()[0])
changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil)
panicOn(err)
PanicOn(err)
if changed != 3000 {
t.Fatalf("expected %v bits changed, got %v", 3000, changed)
}
@ -228,7 +229,7 @@ func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) {
dump := func() {
cur, err := tx.cursor(name)
panicOn(err)
PanicOn(err)
//cur.dump()
_ = cur.tx.dumpAllPages(true)
}
@ -242,7 +243,7 @@ func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) {
itr3 := getRoaringIter(want...)
changed, _, err = tx.ImportRoaringBits(name, itr3, clear, false, rowSize, nil)
panicOn(err)
PanicOn(err)
if changed != expectedBitsChanged {
// cursor_internal_test.go:235: expected 2,724,000 bits changed, got 2,721,000
t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed)
@ -292,7 +293,7 @@ func TestCursor_RoaringImport_auto_many_leaves(t *testing.T) {
//vv("DONE WITH Add()")
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
panicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != expectedBitsChanged {
t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed)
}
@ -300,7 +301,7 @@ func TestCursor_RoaringImport_auto_many_leaves(t *testing.T) {
dump := func() {
cur, err := tx.cursor(name)
panicOn(err)
PanicOn(err)
//cur.dump()
_ = cur.tx.dumpAllPages(true)
}
@ -313,7 +314,7 @@ func TestCursor_RoaringImport_auto_many_leaves(t *testing.T) {
itr2 := getRoaringIter(want...)
changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil)
panicOn(err)
PanicOn(err)
if changed != expectedBitsChanged {
t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed)
}
@ -372,13 +373,13 @@ func TestCursor_putBranchCellsHandlesLotsOfNewBranchesAtTheRoot(t *testing.T) {
t.Fatal(err)
}
c, err := tx.cursor(name)
panicOn(err)
PanicOn(err)
ikeys := itr.ContainerKeys()
for _, ckey := range ikeys {
_, err := c.Seek(ckey)
panicOn(err)
PanicOn(err)
break // after the first seek
}
@ -404,7 +405,7 @@ func TestCursor_putBranchCellsHandlesLotsOfNewBranchesAtTheRoot(t *testing.T) {
branch := branchCell{LeftKey: group[0].Key}
branch.ChildPgno, err = c.tx.allocatePgno()
panicOn(err)
PanicOn(err)
branches = append(branches, branch)
//vv("on group i=%v of %v, group[0].Key = %v; branch.ChildPgno=%v; branch.Key=%v", i, len(groups.slc), int(group[0].Key), (branch.ChildPgno), int(branch.Key))
@ -422,13 +423,13 @@ func TestCursor_putBranchCellsHandlesLotsOfNewBranchesAtTheRoot(t *testing.T) {
}
err = c.tx.writePage(buf[:])
panicOn(err)
PanicOn(err)
}
//vv("branches ckeys = '%#v'", keysFromParents(branches))
err = c.putBranchCells(0, branches)
panicOn(err)
PanicOn(err)
//c.tx.dumpAllPages(true)
}
@ -476,7 +477,7 @@ func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) {
defer tx.Rollback()
dump := func() {
cur, err := tx.cursor(name)
panicOn(err)
PanicOn(err)
//cur.dump()
_ = cur.tx.dumpAllPages(true)
}
@ -487,7 +488,7 @@ func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) {
itr := getRoaringIter(want[i*3000 : (i+1)*3000]...)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
panicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != 3000 {
t.Fatalf("expected %v bits changed, got %v", 3000, changed)
}
@ -505,7 +506,7 @@ func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) {
itr := getRoaringIter(want[i*3000 : (i+1)*3000]...)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
panicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != 3000 {
t.Fatalf("expected %v bits changed, got %v", 3000, changed)
}
@ -559,7 +560,7 @@ func TestCursor_from_B_to_C(t *testing.T) {
defer tx.Rollback()
dump := func() {
cur, err := tx.cursor(name)
panicOn(err)
PanicOn(err)
//cur.dump()
_ = cur.tx.dumpAllPages(true)
}
@ -567,7 +568,7 @@ func TestCursor_from_B_to_C(t *testing.T) {
itr := getRoaringIter(want[:6000]...)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
panicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != 6000 {
t.Fatalf("expected %v bits changed, got %v", 6000, changed)
}
@ -578,7 +579,7 @@ func TestCursor_from_B_to_C(t *testing.T) {
itr = getRoaringIter(want[6000:9000]...)
changed, _, err = tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
panicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != 3000 {
t.Fatalf("expected %v bits changed, got %v", 3000, changed)
}
@ -595,7 +596,7 @@ func TestCursor_from_B_to_C(t *testing.T) {
itr := getRoaringIter(want[i*3000 : (i+1)*3000]...)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
panicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != 3000 {
t.Fatalf("expected %v bits changed, got %v", 3000, changed) // failing here got 0
}

26
rbf/ingest_test.go
View file

@ -29,8 +29,8 @@ import (
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/testhook"
// "github.com/pilosa/pilosa/v2/txkey"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
func rbfName(index, field, view string, shard uint64) string {
@ -74,17 +74,17 @@ func TestIngest_lots_of_views(t *testing.T) {
}()
path, err := testhook.TempDir(t, "rbf_ingest_lots_of_views")
panicOn(err)
PanicOn(err)
defer os.Remove(path)
cfg := cfg.NewDefaultConfig()
db := NewDB(path, cfg)
panicOn(db.Open())
PanicOn(db.Open())
// setup profiling
if false {
profile, err := os.Create("./rbf_ingest_put_ct.cpu")
panicOn(err)
PanicOn(err)
_ = pprof.StartCPUProfile(profile)
defer func() {
pprof.StopCPUProfile()
@ -94,7 +94,7 @@ func TestIngest_lots_of_views(t *testing.T) {
// put containers
tx, err := db.Begin(true)
panicOn(err)
PanicOn(err)
index := "i"
field := "f"
@ -117,30 +117,30 @@ func TestIngest_lots_of_views(t *testing.T) {
view = fmt.Sprintf("view_%v", i)
name := rbfName(index, field, view, shard)
err = tx.PutContainer(name, ckey, ct)
panicOn(err)
PanicOn(err)
ct2, err := tx.Container(name, ckey)
panicOn(err)
PanicOn(err)
if err := ct2.BitwiseCompare(ct); err != nil {
panic("ct2 != ct")
PanicOn("ct2 != ct")
}
// write .dot of it...
if false { //ckey == nCt-1 {
c, err := tx.cursor(name)
if err == ErrBitmapNotFound {
panic("not found")
PanicOn("not found")
} else if err != nil {
panic(err)
PanicOn(err)
}
defer c.Close()
c.Dump("one.bitmap.dot.dump")
}
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
sz, err := DiskUse(path, "")
panicOn(err)
PanicOn(err)
_ = sz
//vv("sz in bytes= %v", sz)
db.Close()
@ -153,7 +153,7 @@ func DiskUse(root string, requiredSuffix string) (tot int, err error) {
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if info == nil {
panic(fmt.Sprintf("info was nil for path = '%v'", path))
PanicOn(fmt.Sprintf("info was nil for path = '%v'", path))
}
if info.IsDir() {
// skip the size of directories themselves, only summing files.

37
rbf/rbf.go
View file

@ -30,6 +30,7 @@ import (
"github.com/benbjohnson/immutable"
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/shardwidth"
. "github.com/pilosa/pilosa/v2/vprint"
)
const (
@ -355,8 +356,9 @@ func (c *leafCell) Bitmap(tx *Tx) []uint64 {
_, bm, _ := tx.leafCellBitmap(toPgno(c.Data))
return bm
default:
panic(fmt.Sprintf("invalid container type: %d", c.Type))
PanicOn(fmt.Errorf("invalid container type: %d", c.Type))
}
return nil
}
// Values returns a slice of 16-bit values from a container.
@ -382,8 +384,9 @@ func (c *leafCell) Values(tx *Tx) []uint16 {
case ContainerTypeNone:
return []uint16{}
default:
panic(fmt.Sprintf("invalid container type: %d", c.Type))
PanicOn(fmt.Errorf("invalid container type: %d", c.Type))
}
return nil
}
func bitmapValues(bm []uint64) []uint16 {
@ -409,7 +412,7 @@ func (c *leafCell) firstValue(tx *Tx) uint16 {
return r[0].Start
case ContainerTypeBitmapPtr:
_, slc, err := tx.leafCellBitmap(toPgno(c.Data))
panicOn(err)
PanicOn(err)
for i, v := range slc {
for j := uint(0); j < 64; j++ {
if v&(1<<j) != 0 {
@ -417,10 +420,12 @@ func (c *leafCell) firstValue(tx *Tx) uint16 {
}
}
}
panic(fmt.Sprintf("rbf.leafCell.firstValue(): no values set in bitmap container: key=%d", c.Key))
PanicOn(fmt.Errorf("rbf.leafCell.firstValue(): no values set in bitmap container: key=%d", c.Key))
default:
panic(fmt.Sprintf("invalid container type: %d", c.Type))
PanicOn(fmt.Errorf("invalid container type: %d", c.Type))
}
return 0
}
// helper for lastValue()
@ -432,7 +437,8 @@ func (c *leafCell) lastValueFromBitmap(a []uint64) uint16 {
}
}
}
panic(fmt.Sprintf("rbf.leafCell.lastValueFromBitmap(): no values set in bitmap container: key=%d", c.Key))
PanicOn(fmt.Errorf("rbf.leafCell.lastValueFromBitmap(): no values set in bitmap container: key=%d", c.Key))
return 0
}
// lastValue the last value from the container.
@ -451,11 +457,12 @@ func (c *leafCell) lastValue(tx *Tx) uint16 {
case ContainerTypeBitmapPtr:
_, a, err := tx.leafCellBitmap(toPgno(c.Data))
panicOn(err)
PanicOn(err)
return c.lastValueFromBitmap(a)
default:
panic(fmt.Sprintf("invalid container type: %d", c.Type))
PanicOn(fmt.Errorf("invalid container type: %d", c.Type))
}
return 0
}
// countRange returns the bit count within the given range.
@ -477,11 +484,12 @@ func (c *leafCell) countRange(tx *Tx, start, end int32) (n int) {
return int(roaring.BitmapCountRange(toArray64(c.Data), start, end))
case ContainerTypeBitmapPtr:
_, a, err := tx.leafCellBitmap(toPgno(c.Data))
panicOn(err)
PanicOn(err)
return int(roaring.BitmapCountRange(a, start, end))
default:
panic(fmt.Sprintf("invalid container type: %d", c.Type))
PanicOn(fmt.Errorf("invalid container type: %d", c.Type))
}
return
}
func readLeafCellKey(page []byte, i int) uint64 {
@ -560,8 +568,9 @@ func readLeafCellBytesAtOffset(page []byte, offset int) []byte {
case ContainerTypeBitmapPtr:
return buf[:leafCellHeaderSize+4]
default:
panic(fmt.Sprintf("invalid cell type: %d", typ))
PanicOn(fmt.Errorf("invalid cell type: %d", typ))
}
return nil
}
// leafPageSize returns the number of bytes used on a leaf page.
@ -715,13 +724,13 @@ func Walk(tx *Tx, pgno uint32, v func(uint32, []*RootRecord)) {
for pgno := readMetaRootRecordPageNo(tx.meta[:]); pgno != 0; {
page, _, err := tx.readPage(pgno)
if err != nil {
panic(err)
PanicOn(err)
}
// Read all records on the page.
a, err := readRootRecords(page)
if err != nil {
panic(err)
PanicOn(err)
}
v(pgno, a)
// Read next overflow page number.
@ -731,7 +740,7 @@ func Walk(tx *Tx, pgno uint32, v func(uint32, []*RootRecord)) {
func assert(condition bool) {
if !condition {
panic("assertion failed")
PanicOn(fmt.Errorf("assertion failed"))
}
}

44
rbf/tx.go
View file

@ -26,7 +26,7 @@ import (
"github.com/pilosa/pilosa/v2/hash"
"github.com/pilosa/pilosa/v2/roaring"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
//txkey "github.com/pilosa/pilosa/v2/txkey"
. "github.com/pilosa/pilosa/v2/vprint"
)
var _ = txkey.ToString
@ -135,7 +135,7 @@ func (tx *Tx) Rollback() {
// Disconnect transaction from DB.
tx.db.mu.Lock()
defer tx.db.mu.Unlock()
panicOn(tx.db.removeTx(tx))
PanicOn(tx.db.removeTx(tx))
}
// Root returns the root page number for a bitmap. Returns 0 if the bitmap does not exist.
@ -919,7 +919,7 @@ func (tx *Tx) allocatePgno() (uint32, error) {
if changed, err := c.Remove(uint64(pgno)); err != nil {
return 0, err
} else if !changed {
panic(fmt.Sprintf("tx.Tx.allocatePgno(): double alloc: %d", pgno))
PanicOn(fmt.Sprintf("tx.Tx.allocatePgno(): double alloc: %d", pgno))
}
return pgno, nil
}
@ -960,7 +960,7 @@ func (tx *Tx) freePgno(pgno uint32) error {
if changed, err := c.Add(uint64(pgno)); err != nil {
return err
} else if !changed {
panic(fmt.Sprintf("rbf.Tx.freePgno(): double free: %d", pgno))
PanicOn(fmt.Sprintf("rbf.Tx.freePgno(): double free: %d", pgno))
}
return nil
}
@ -1200,7 +1200,7 @@ func (tx *Tx) ForEachRange(name string, start, end uint64, fn func(uint64) error
}
}
default:
panic(fmt.Sprintf("invalid container type: %d", cell.Type))
PanicOn(fmt.Sprintf("invalid container type: %d", cell.Type))
}
}
}
@ -1299,7 +1299,7 @@ func (tx *Tx) Min(name string) (uint64, bool, error) {
func (tx *Tx) UnionInPlace(name string, others ...*roaring.Bitmap) error {
rbm, err := tx.RoaringBitmap(name)
panicOn(err)
PanicOn(err)
rbm.UnionInPlace(others...)
// iterate over the containers that changed within rbm, and write them back to disk.
@ -1313,7 +1313,7 @@ func (tx *Tx) UnionInPlace(name string, others ...*roaring.Bitmap) error {
// TODO: only write the changed ones back, as optimization?
// Compare to ImportRoaringBits.
err := tx.PutContainer(name, containerKey, rc)
panicOn(err)
PanicOn(err)
}
return nil
}
@ -1395,11 +1395,11 @@ func (tx *Tx) CountRange(name string, start, end uint64) (uint64, error) {
func (tx *Tx) OffsetRange(name string, offset, start, endx uint64) (*roaring.Bitmap, error) {
if lowbits(offset) != 0 {
panic("offset must not contain low bits")
PanicOn("offset must not contain low bits")
} else if lowbits(start) != 0 {
panic("range start must not contain low bits")
PanicOn("range start must not contain low bits")
} else if lowbits(endx) != 0 {
panic("range endx must not contain low bits")
PanicOn("range endx must not contain low bits")
}
tx.mu.RLock()
@ -1532,7 +1532,8 @@ func (si *emptyContainerIterator) Next() bool {
return false
}
func (si *emptyContainerIterator) Value() (uint64, *roaring.Container) {
panic("emptyContainerIterator never has any Values")
PanicOn("emptyContainerIterator never has any Values")
return 0, nil
}
func (tx *Tx) Dump(short bool, shard uint64) {
@ -1544,14 +1545,14 @@ func (tx *Tx) DumpString(short bool, shard uint64) (r string) {
// grab root records, for a list of bitmaps.
records, err := tx.RootRecords()
panicOn(err)
PanicOn(err)
n := 0
for itr := records.Iterator(); !itr.Done(); {
name, _ := itr.Next()
c, err := tx.cursor(name.(string))
panicOn(err)
PanicOn(err)
defer c.Close()
err = c.First() // First will rewind to beginning.
@ -1560,17 +1561,17 @@ func (tx *Tx) DumpString(short bool, shard uint64) (r string) {
n++
continue
}
panicOn(err)
PanicOn(err)
for {
err := c.Next()
if err == io.EOF {
break
}
panicOn(err)
PanicOn(err)
elem := &c.stack.elems[c.stack.top]
leafPage, _, err := c.tx.readPage(elem.pgno)
panicOn(err)
PanicOn(err)
cell := readLeafCell(leafPage, elem.index)
ckey := cell.Key
@ -1595,7 +1596,7 @@ func containerToBytes(ct *roaring.Container) []byte {
ty := roaring.ContainerType(ct)
switch ty {
case roaring.ContainerNil:
panic("nil container")
PanicOn("nil container")
case roaring.ContainerArray:
return fromArray16(roaring.AsArray(ct))
case roaring.ContainerBitmap:
@ -1603,7 +1604,8 @@ func containerToBytes(ct *roaring.Container) []byte {
case roaring.ContainerRun:
return fromInterval16(roaring.AsRuns(ct))
}
panic(fmt.Sprintf("unknown container type '%v'", int(ty)))
PanicOn(fmt.Sprintf("unknown container type '%v'", int(ty)))
return nil
}
func bitmapAsString(rbm *roaring.Bitmap) (r string) {
@ -1785,7 +1787,7 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear
err = tx.putContainerWithCursor(cur, itrKey, newC)
if err != nil {
panicOn(err)
PanicOn(err)
return
}
continue
@ -1927,7 +1929,7 @@ func (tx *Tx) Pages(pgnos []uint32) ([]Page, error) {
pages = append(pages, &FreePage{FreePageInfo: info})
default:
panic(fmt.Sprintf("invalid page info type %T", info))
PanicOn(fmt.Sprintf("invalid page info type %T", info))
}
}
@ -2047,7 +2049,7 @@ func (tx *Tx) walkPageInfo(infos []PageInfo, root uint32, name string) error {
Tree: name,
}
default:
panic(fmt.Sprintf("unexpected page type %d for page %d", typ, pgno))
PanicOn(fmt.Sprintf("unexpected page type %d for page %d", typ, pgno))
}
return nil

13
rbf/util.go
View file

@ -19,6 +19,7 @@ import (
"strings"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
. "github.com/pilosa/pilosa/v2/vprint"
)
func (tx *Tx) dumpAllPages(showLeaves bool) error {
@ -56,9 +57,9 @@ func (tx *Tx) dumpAllPages(showLeaves bool) error {
fmt.Printf("next=%d\n", info.Next)
page, _, err := tx.readPage(uint32(pgno))
panicOn(err)
PanicOn(err)
rootRecords, err := readRootRecords(page)
panicOn(err)
PanicOn(err)
for k, rr := range rootRecords {
fmt.Printf(" [%02v] Name:'%v' pgno:%v\n", k, prefixToString(rr.Name), rr.Pgno)
}
@ -73,7 +74,7 @@ func (tx *Tx) dumpAllPages(showLeaves bool) error {
fmt.Printf("flags=x%x,celln=%d\n", info.Flags, info.CellN)
page, _, err := tx.readPage(uint32(pgno))
panicOn(err)
PanicOn(err)
var leafCells [PageSize / 8]leafCell
cells := readLeafCells(page, leafCells[:])
@ -88,7 +89,7 @@ func (tx *Tx) dumpAllPages(showLeaves bool) error {
fmt.Printf("flags=x%x,celln=%d\n", info.Flags, info.CellN)
page, _, err := tx.readPage(uint32(pgno))
panicOn(err)
PanicOn(err)
cells := readBranchCells(page)
for i, cell := range cells {
@ -240,12 +241,12 @@ func (c_orig *Cursor) debugStringBitmaps() (r string) {
if err == io.EOF {
break
}
panicOn(err)
PanicOn(err)
//instead of cell := c2.cell()
elem := &c2.stack.elems[c2.stack.top]
leafPage, _, err := c2.tx.readPage(elem.pgno)
panicOn(err)
PanicOn(err)
cell := readLeafCell(leafPage, elem.index)
ckey := cell.Key

208
rbf/vprint.go
View file

@ -1,208 +0,0 @@
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package rbf
import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"runtime"
"runtime/debug"
"strings"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("\n%s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}
var _ = stack // happy linter
var _ = listFilesUnderDir
// listFilesUnderDir returns the paths of files found under directory root.
// If includeRoot is true, it returns the full path, otherwise paths are relative to root.
// If requriedSuffix is supplied, the returned file paths will end in that,
// and any other files found during the walk of the directory tree will be ignored.
// If ignoreEmpty is true, files of size 0 will be excluded.
func listFilesUnderDir(root string, includeRoot bool, requiredSuffix string, ignoreEmpty bool) (files []string, err error) {
if !DirExists(root) {
return nil, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root)
}
n := len(root) + 1
if includeRoot {
n = 0
}
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if len(path) < n {
// ignore
} else {
if info == nil {
panic(fmt.Sprintf("info was nil for path = '%v'", path))
}
if info.IsDir() {
// skip directories.
} else {
if ignoreEmpty && info.Size() == 0 {
return nil
}
if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) {
files = append(files, path[n:])
}
}
}
return nil
})
return
}

6
rrtx.go
View file

@ -30,7 +30,7 @@ import (
txkey "github.com/pilosa/pilosa/v2/short_txkey"
"github.com/pilosa/pilosa/v2/storage"
//txkey "github.com/pilosa/pilosa/v2/txkey"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
)
@ -120,7 +120,7 @@ func (tx *RoaringTx) Pointer() string {
// the transaction Commits or Rollsback.
func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
b, err := tx.bitmap(index, field, view, shard)
panicOn(err)
PanicOn(err)
return b.Iterator()
}
@ -150,7 +150,7 @@ func (tx *RoaringTx) Readonly() bool {
func (tx *RoaringTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
frag, err := tx.getFragment(index, field, view, shard)
panicOn(err)
PanicOn(err)
frag.incrementOpN(changedN)
}

14
rrtx_internal_test.go
View file

@ -16,22 +16,24 @@ package pilosa
import (
"testing"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
func TestRoaring_HasData(t *testing.T) {
holder := newHolderWithTempPath(t, "roaring")
idx, err := holder.CreateIndex("i", IndexOptions{})
panicOn(err)
PanicOn(err)
defer idx.Close()
db, err := globalRoaringReg.OpenDBWrapper(idx.path, false, nil)
panicOn(err)
PanicOn(err)
db.SetHolder(idx.holder)
// HasData should start out false.
hasAnything, err := db.HasData()
panicOn(err)
PanicOn(err)
if hasAnything {
t.Fatalf("HasData reported existing data on an empty database")
@ -45,10 +47,10 @@ func TestRoaring_HasData(t *testing.T) {
defer tx.Rollback()
f, err := idx.CreateField(field)
panicOn(err)
PanicOn(err)
_, err = f.SetBit(tx, 1, 1, nil)
panicOn(err)
panicOn(tx.Commit())
PanicOn(err)
PanicOn(tx.Commit())
hasAnything, err = db.HasData()
if err != nil {

2
server/handler_test.go
View file

@ -1534,7 +1534,7 @@ func TestCluster_TranslateStore(t *testing.T) {
if err := cluster.GetIdleNode(0).Start(); err != nil {
t.Fatalf("starting node 0: %v", err)
}
defer cluster.GetIdleNode(0).Close()
defer cluster.GetIdleNode(0).Close() // nolint: errcheck
test.Do(t, "POST", cluster.GetIdleNode(0).URL()+"/index/i0", "{\"options\": {\"keys\": true}}")
}

88
stattx.go
View file

@ -26,6 +26,7 @@ import (
"github.com/pilosa/pilosa/v2/debugstats"
"github.com/pilosa/pilosa/v2/roaring"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
// statTx is useful to profile on a
@ -242,7 +243,8 @@ func (k kall) String() string {
case kUseRowCache:
return "kUseRowCache"
}
panic(fmt.Sprintf("unknown kall '%v'", int(k)))
PanicOn(fmt.Sprintf("unknown kall '%v'", int(k)))
return ""
}
var _ = newStatTx // happy linter
@ -293,8 +295,8 @@ func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
@ -307,8 +309,8 @@ func (c *statTx) Dump(short bool, shard uint64) {
func (c *statTx) Readonly() bool {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Readonly() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Readonly() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Readonly()
@ -327,8 +329,8 @@ func (c *statTx) Rollback() {
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Rollback() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
c.b.Rollback()
@ -344,8 +346,8 @@ func (c *statTx) Commit() error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Commit() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Commit()
@ -361,8 +363,8 @@ func (c *statTx) RoaringBitmap(index, field, view string, shard uint64) (*roarin
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmap() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.RoaringBitmap(index, field, view, shard)
@ -378,8 +380,8 @@ func (c *statTx) Container(index, field, view string, shard uint64, key uint64)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Container() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Container(index, field, view, shard, key)
@ -395,8 +397,8 @@ func (c *statTx) PutContainer(index, field, view string, shard uint64, key uint6
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see PutContainer() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.PutContainer(index, field, view, shard, key, rc)
@ -412,8 +414,8 @@ func (c *statTx) RemoveContainer(index, field, view string, shard uint64, key ui
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RemoveContainer() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.RemoveContainer(index, field, view, shard, key)
@ -437,8 +439,8 @@ func (c *statTx) Add(index, field, view string, shard uint64, batched bool, a ..
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Add() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Add() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Add(index, field, view, shard, batched, a...)
@ -454,8 +456,8 @@ func (c *statTx) Remove(index, field, view string, shard uint64, a ...uint64) (c
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Remove() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Remove(index, field, view, shard, a...)
@ -471,8 +473,8 @@ func (c *statTx) Contains(index, field, view string, shard uint64, key uint64) (
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Contains() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Contains(index, field, view, shard, key)
@ -488,8 +490,8 @@ func (c *statTx) ContainerIterator(index, field, view string, shard uint64, firs
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ContainerIterator() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey)
@ -509,8 +511,8 @@ func (c *statTx) ForEach(index, field, view string, shard uint64, fn func(i uint
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEach() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.ForEach(index, field, view, shard, fn)
@ -526,8 +528,8 @@ func (c *statTx) ForEachRange(index, field, view string, shard uint64, start, en
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ForEachRange() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.ForEachRange(index, field, view, shard, start, end, fn)
@ -543,8 +545,8 @@ func (c *statTx) Count(index, field, view string, shard uint64) (uint64, error)
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Count() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Count(index, field, view, shard)
@ -560,8 +562,8 @@ func (c *statTx) Max(index, field, view string, shard uint64) (uint64, error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Max() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Max(index, field, view, shard)
@ -577,8 +579,8 @@ func (c *statTx) Min(index, field, view string, shard uint64) (uint64, bool, err
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Min() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Min(index, field, view, shard)
@ -594,8 +596,8 @@ func (c *statTx) UnionInPlace(index, field, view string, shard uint64, others ..
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see UnionInPlace() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.UnionInPlace(index, field, view, shard, others...)
@ -611,8 +613,8 @@ func (c *statTx) CountRange(index, field, view string, shard uint64, start, end
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see CountRange() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.CountRange(index, field, view, shard, start, end)
@ -627,8 +629,8 @@ func (c *statTx) OffsetRange(index, field, view string, shard, offset, start, en
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see OffsetRange() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see OffsetRange() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.OffsetRange(index, field, view, shard, offset, start, end)
@ -644,8 +646,8 @@ func (c *statTx) RoaringBitmapReader(index, field, view string, shard uint64, fr
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmapReader() panic '%v' at '%v'", r, stack())
panic(r)
AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)

22
synthload/synthload_test.go
View file

@ -49,12 +49,12 @@ func Test_SynthLoad_ImportSchema(t *testing.T) {
// get the holder.Path to write to
h := m0.API.Holder()
target := h.Path()
panicOn(h.Close())
PanicOn(h.Close())
panicOn(unpackTarball(tarball, target))
PanicOn(unpackTarball(tarball, target))
// reopen
panicOn(h.Open())
PanicOn(h.Open())
qs := strings.Split(pql, "\n\n")
//vv("qs = '%#v'", qs)
@ -92,7 +92,7 @@ func Test_SynthLoad_ImportSchema(t *testing.T) {
}
qr, err := m0.API.Query(context.Background(), req)
panicOn(err)
PanicOn(err)
vv("qr = '%#v'", qr)
}
}
@ -103,22 +103,22 @@ func applySchema(m0 *test.Command, schemaStr string) {
// don't need schema now that we import the tarball, it has it all.
schema := &pilosa.Schema{}
err := json.NewDecoder(bytes.NewBufferString(schemaStr)).Decode(schema)
panicOn(err)
PanicOn(err)
ctx := context.Background()
remote := false
err = m0.API.ApplySchema(ctx, schema, remote)
panicOn(err)
PanicOn(err)
}
func unpackTarball(tarball, target string) error {
vv("target = '%v'", target)
fd, err := os.Open(tarball)
panicOn(err)
PanicOn(err)
defer fd.Close()
gz, err := gzip.NewReader(fd)
panicOn(err)
PanicOn(err)
defer gz.Close()
tarReader := tar.NewReader(gz)
@ -134,16 +134,16 @@ func unpackTarball(tarball, target string) error {
info := header.FileInfo()
if info.IsDir() {
if err = os.MkdirAll(path, info.Mode()); err != nil {
panicOn(err)
PanicOn(err)
}
continue
}
file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
panicOn(err)
PanicOn(err)
_, err = io.Copy(file, tarReader)
panicOn(err)
PanicOn(err)
file.Close()
}
return nil

170
synthload/vprint.go
View file

@ -1,170 +0,0 @@
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package synthload
import (
"fmt"
"io"
"os"
"path"
"runtime"
"runtime/debug"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
var _ = panicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("\n%s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}
var _ = stack
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}

10
test/holder.go
View file

@ -23,11 +23,10 @@ import (
"github.com/pilosa/pilosa/v2/boltdb"
"github.com/pilosa/pilosa/v2/pql"
"github.com/pilosa/pilosa/v2/testhook"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
)
var panicOn = pilosa.PanicOn
// Holder is a test wrapper for pilosa.Holder.
type Holder struct {
*pilosa.Holder
@ -148,9 +147,6 @@ func (h *Holder) SetBit(index, field string, rowID, columnID uint64) {
h.SetBitTime(index, field, rowID, columnID, nil)
}
var vv = pilosa.VV
var _ = vv // happy linter
// SetBitTime sets a bit with timestamp on the given field.
func (h *Holder) SetBitTime(index, field string, rowID, columnID uint64, t *time.Time) {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
@ -167,7 +163,7 @@ func (h *Holder) SetBitTime(index, field string, rowID, columnID uint64, t *time
if err != nil {
panic(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
}
// ClearBit clears a bit on the given field.
@ -186,7 +182,7 @@ func (h *Holder) ClearBit(index, field string, rowID, columnID uint64) {
if err != nil {
panic(err)
}
panicOn(tx.Commit())
PanicOn(tx.Commit())
}
// MustSetBits sets columns on a row. Panic on error.

2
translate.go
View file

@ -233,7 +233,7 @@ func NewMultiTranslateEntryReader(ctx context.Context, readers []TranslateEntryR
func (r *MultiTranslateEntryReader) Close() error {
r.cancel()
for i := range r.readers {
r.readers[i].Close()
r.readers[i].Close() // nolint: errcheck
}
r.wg.Wait()
return nil

27
tx_test.go
View file

@ -25,12 +25,13 @@ import (
"github.com/pilosa/pilosa/v2/server"
"github.com/pilosa/pilosa/v2/storage"
"github.com/pilosa/pilosa/v2/test"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
func queryIRABit(m0api *pilosa.API, acctOwnerID uint64, iraField string, iraRowID uint64, index string) (bit bool) {
query := fmt.Sprintf("Row(%v=%v)", iraField, iraRowID) // acctOwnerID)
res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
panicOn(err)
PanicOn(err)
cols := res.Results[0].(*pilosa.Row).Columns()
for i := range cols {
if cols[i] == acctOwnerID {
@ -43,7 +44,7 @@ func queryIRABit(m0api *pilosa.API, acctOwnerID uint64, iraField string, iraRowI
func mustQueryAcct(m0api *pilosa.API, acctOwnerID uint64, fieldAcct0, index string) (acctBal int64) {
query := fmt.Sprintf("FieldValue(field=%v, column=%v)", fieldAcct0, acctOwnerID)
res, err := m0api.Query(context.Background(), &pilosa.QueryRequest{Index: index, Query: query})
panicOn(err)
PanicOn(err)
if len(res.Results) == 0 {
return 0
@ -171,17 +172,17 @@ func TestAPI_ImportAtomicRecord(t *testing.T) {
iraBit := queryIRABit(m0api, acctOwnerID, iraField, iraRowID, index)
if !iraBit {
panic("IRA bit should have been set")
PanicOn("IRA bit should have been set")
}
startingBalanceAcct0, startingBalanceAcct1 := queryBalances(m0api, acctOwnerID, fieldAcct0, fieldAcct1, index)
//vv("starting balance: acct0=%v, acct1=%v", startingBalanceAcct0, startingBalanceAcct1)
if startingBalanceAcct0 != expectedBalStartingAcct0 {
panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", expectedBalStartingAcct0, startingBalanceAcct0))
PanicOn(fmt.Sprintf("expected %v, observed %v starting acct0 balance", expectedBalStartingAcct0, startingBalanceAcct0))
}
if startingBalanceAcct1 != expectedBalStartingAcct1 {
panic(fmt.Sprintf("expected %v, observed %v starting acct1 balance", expectedBalStartingAcct1, startingBalanceAcct1))
PanicOn(fmt.Sprintf("expected %v, observed %v starting acct1 balance", expectedBalStartingAcct1, startingBalanceAcct1))
}
//vv("sad path: transferUSD %v from %v -> %v, with power loss half-way through", transferUSD, fieldAcct0, fieldAcct1)
@ -200,7 +201,7 @@ func TestAPI_ImportAtomicRecord(t *testing.T) {
err = m0api.ImportAtomicRecord(ctx, qcx, air, opt)
//err = m0api.ImportAtomicRecord(ctx, nil, air, opt)
if err != pilosa.ErrAborted {
panic(fmt.Sprintf("expected ErrTxnAborted but got err='%#v'", err))
PanicOn(fmt.Sprintf("expected ErrTxnAborted but got err='%#v'", err))
}
// sad path, cleanup
qcx.Abort()
@ -210,10 +211,10 @@ func TestAPI_ImportAtomicRecord(t *testing.T) {
//vv("after power failure tx, balance: acct0=%v, acct1=%v", b0, b1)
if b0 != expectedBalStartingAcct0 {
panic(fmt.Sprintf("expected %v, observed %v starting acct0 balance", expectedBalStartingAcct0, b0))
PanicOn(fmt.Sprintf("expected %v, observed %v starting acct0 balance", expectedBalStartingAcct0, b0))
}
if b1 != expectedBalStartingAcct1 {
panic(fmt.Sprintf("expected %v, observed %v starting acct1 balance", expectedBalStartingAcct1, b1))
PanicOn(fmt.Sprintf("expected %v, observed %v starting acct1 balance", expectedBalStartingAcct1, b1))
}
//vv("good: with power loss half-way, no change in account balances; acct0=%v; acct1=%v", b0, b1)
@ -223,14 +224,14 @@ func TestAPI_ImportAtomicRecord(t *testing.T) {
// happy path with no power failure half-way through.
err = m0api.ImportAtomicRecord(ctx, nil, air)
panicOn(err)
PanicOn(err)
eb0, eb1 := queryBalances(m0api, acctOwnerID, fieldAcct0, fieldAcct1, index)
// should have been applied this time.
if eb0 != expectedBalEndingAcct0 ||
eb1 != expectedBalEndingAcct1 {
panic(fmt.Sprintf("problem: transaction did not get committed/applied. transferUSD=%v, but we see: startingBalanceAcct0=%v -> endingBalanceAcct0=%v; startingBalanceAcct1=%v -> endingBalanceAcct1=%v", transferUSD, startingBalanceAcct0, eb0, startingBalanceAcct1, eb1))
PanicOn(fmt.Sprintf("problem: transaction did not get committed/applied. transferUSD=%v, but we see: startingBalanceAcct0=%v -> endingBalanceAcct0=%v; startingBalanceAcct1=%v -> endingBalanceAcct1=%v", transferUSD, startingBalanceAcct0, eb0, startingBalanceAcct1, eb1))
}
//vv("ending balance: acct0=%v, acct1=%v", eb0, eb1)
@ -240,18 +241,18 @@ func TestAPI_ImportAtomicRecord(t *testing.T) {
air.Ir[0].Clear = true
err = m0api.ImportAtomicRecord(ctx, nil, air)
panicOn(err)
PanicOn(err)
eb0, eb1 = queryBalances(m0api, acctOwnerID, fieldAcct0, fieldAcct1, index)
if eb0 != 0 ||
eb1 != 0 {
panic("problem: bits did not clear")
PanicOn("problem: bits did not clear")
}
//vv("cleared balances: acct0=%v, acct1=%v", eb0, eb1)
iraBit = queryIRABit(m0api, acctOwnerID, iraField, iraRowID, index)
if iraBit {
panic("IRA bit should have been cleared")
PanicOn("IRA bit should have been cleared")
}
}

90
txfactory.go
View file

@ -31,6 +31,7 @@ import (
"github.com/pilosa/pilosa/v2/roaring"
txkey "github.com/pilosa/pilosa/v2/short_txkey"
"github.com/pilosa/pilosa/v2/storage"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
"github.com/zeebo/blake3"
)
@ -140,7 +141,7 @@ func (q *Qcx) Finish() (err error) {
defer q.mu.Unlock()
if q.RequiredForAtomicWriteTx != nil {
if q.RequiredTxo.Write {
err = (*q.RequiredForAtomicWriteTx).Commit() // panic here on 2nd. is this a double commit?
err = (*q.RequiredForAtomicWriteTx).Commit() // PanicOn here on 2nd. is this a double commit?
} else {
(*q.RequiredForAtomicWriteTx).Rollback()
}
@ -173,7 +174,7 @@ func (q *Qcx) Reset() {
q.mu.Lock()
defer q.mu.Unlock()
if !q.done {
panic("must call Qcx.Abort() or Qcx.Finish() before calling Reset().")
PanicOn("must call Qcx.Abort() or Qcx.Finish() before calling Reset().")
}
q.unprotected_reset()
}
@ -270,16 +271,16 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) {
// verify that shard and index match!
ro := qcx.RequiredTxo
if o.Shard != ro.Shard {
panic(fmt.Sprintf("shard mismatch: o.Shard = %v while qcx.RequiredTxo.Shard = %v", o.Shard, ro.Shard))
PanicOn(fmt.Sprintf("shard mismatch: o.Shard = %v while qcx.RequiredTxo.Shard = %v", o.Shard, ro.Shard))
}
if o.Index == nil {
panic("o.Index annot be nil")
PanicOn("o.Index annot be nil")
}
if ro.Index == nil {
panic("ro.Index annot be nil")
PanicOn("ro.Index annot be nil")
}
if o.Index.name != ro.Index.name {
panic(fmt.Sprintf("index mismatch: o.Index = %v while qcx.RequiredTxo.Index = %v", o.Index.name, ro.Index.name))
PanicOn(fmt.Sprintf("index mismatch: o.Index = %v while qcx.RequiredTxo.Index = %v", o.Index.name, ro.Index.name))
}
return *qcx.RequiredForAtomicWriteTx, NoopFinisher, nil
}
@ -311,7 +312,7 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) {
// so defer finisher(nil) means always Commit writes, ignoring
// the enclosing functions return status.
if perr == nil || *perr == nil {
panicOn(tx.Commit())
PanicOn(tx.Commit())
} else {
tx.Rollback()
}
@ -330,7 +331,7 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) {
// to this shard/index will re-use it.
func (qcx *Qcx) StartAtomicWriteTx(o Txo) {
if !o.Write {
panic("must have o.Write true")
PanicOn("must have o.Write true")
}
qcx.mu.Lock()
defer qcx.mu.Unlock()
@ -349,22 +350,22 @@ func (qcx *Qcx) StartAtomicWriteTx(o Txo) {
// verify that shard and index match!
ro := qcx.RequiredTxo
if o.Shard != ro.Shard {
panic(fmt.Sprintf("shard mismatch: o.Shard = %v while qcx.RequiredTxo.Shard = %v", o.Shard, ro.Shard))
PanicOn(fmt.Sprintf("shard mismatch: o.Shard = %v while qcx.RequiredTxo.Shard = %v", o.Shard, ro.Shard))
}
if o.Index == nil {
panic("o.Index annot be nil")
PanicOn("o.Index annot be nil")
}
if ro.Index == nil {
panic("ro.Index annot be nil")
PanicOn("ro.Index annot be nil")
}
if o.Index.name != ro.Index.name {
panic(fmt.Sprintf("index mismatch: o.Index = %v while qcx.RequiredTxo.Index = %v", o.Index.name, ro.Index.name))
PanicOn(fmt.Sprintf("index mismatch: o.Index = %v while qcx.RequiredTxo.Index = %v", o.Index.name, ro.Index.name))
}
}
func (qcx *Qcx) SetRequiredForAtomicWriteTx(tx Tx) {
if tx == nil || NilInside(tx) {
panic("cannot set nil tx in SetRequiredForAtomicWriteTx")
PanicOn("cannot set nil tx in SetRequiredForAtomicWriteTx")
}
qcx.mu.Lock()
qcx.RequiredForAtomicWriteTx = &tx
@ -436,7 +437,8 @@ func (ty txtype) DirectoryName() string {
case boltTxn:
return "boltdb"
}
panic(fmt.Sprintf("unkown txtype %v", int(ty)))
PanicOn(fmt.Sprintf("unkown txtype %v", int(ty)))
return ""
}
func (txf *TxFactory) NeedsSnapshot() (b bool) {
@ -455,7 +457,7 @@ func MustBackendToTxtype(backend string) (types []txtype) {
if strings.Contains(backend, "_") {
srcs = strings.Split(backend, "_")
if len(srcs) != 2 {
panic("only two blue-green comparisons permitted")
PanicOn("only two blue-green comparisons permitted")
}
} else {
srcs = append(srcs, backend)
@ -470,11 +472,11 @@ func MustBackendToTxtype(backend string) (types []txtype) {
case BoltTxn: // "bolt"
types = append(types, boltTxn)
default:
panic(fmt.Sprintf("unknown backend '%v'", s))
PanicOn(fmt.Sprintf("unknown backend '%v'", s))
}
if i == 1 {
if types[1] == types[0] {
panic(fmt.Sprintf("cannot blue-green the same backend on both arms: '%v'", s))
PanicOn(fmt.Sprintf("cannot blue-green the same backend on both arms: '%v'", s))
}
}
}
@ -796,7 +798,7 @@ type grpkey struct {
func mustHaveIndexShard(o *Txo) {
if o.Index == nil || o.Index.name == "" {
panic("index must be set on Txo")
PanicOn("index must be set on Txo")
}
}
@ -843,10 +845,10 @@ func (g *TxGroup) AddTx(tx Tx) {
g.mu.Lock()
defer g.mu.Unlock()
if g.finished {
panic("in TxGroup.Finish(): TxGroup already finished")
PanicOn("in TxGroup.Finish(): TxGroup already finished")
}
if NilInside(tx) {
panic("Cannot add nil Tx to TxGroup")
PanicOn("Cannot add nil Tx to TxGroup")
}
if tx.Readonly() {
@ -860,7 +862,7 @@ func (g *TxGroup) AddTx(tx Tx) {
key := grpkey{write: o.Write, index: o.Index.name, shard: o.Shard}
prior, ok := g.all[key]
if ok {
panic(fmt.Sprintf("already have Tx in group for this, we should have re-used it! prior is '%v'; tx='%v'", prior, tx))
PanicOn(fmt.Sprintf("already have Tx in group for this, we should have re-used it! prior is '%v'; tx='%v'", prior, tx))
}
g.all[key] = tx
}
@ -872,7 +874,7 @@ func (g *TxGroup) FinishGroup() (err error) {
g.mu.Lock()
defer g.mu.Unlock()
if g.finished {
panic("in TxGroup.Finish(): TxGroup already finished")
PanicOn("in TxGroup.Finish(): TxGroup already finished")
}
g.finished = true
for i, tx := range g.writes {
@ -930,20 +932,20 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) {
if o.Fragment != nil {
if o.Fragment.index() != indexName {
panic(fmt.Sprintf("inconsistent NewTx request: o.Fragment.index='%v' but indexName='%v'", o.Fragment.index(), indexName))
PanicOn(fmt.Sprintf("inconsistent NewTx request: o.Fragment.index='%v' but indexName='%v'", o.Fragment.index(), indexName))
}
if o.Fragment.shard != o.Shard {
panic(fmt.Sprintf("inconsistent NewTx request: o.Fragment.shard='%v' but o.Shard='%v'", o.Fragment.shard, o.Shard))
PanicOn(fmt.Sprintf("inconsistent NewTx request: o.Fragment.shard='%v' but o.Shard='%v'", o.Fragment.shard, o.Shard))
}
}
// look up in the collection of open databases, and get our
// per-shard database. Opens a new one if needed.
dbs, err := f.dbPerShard.GetDBShard(indexName, o.Shard, o.Index)
panicOn(err)
PanicOn(err)
if dbs.Shard != o.Shard {
panic(fmt.Sprintf("asked for o.Shard=%v but got dbs.Shard=%v", int(o.Shard), int(dbs.Shard)))
PanicOn(fmt.Sprintf("asked for o.Shard=%v but got dbs.Shard=%v", int(o.Shard), int(dbs.Shard)))
}
//vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.types='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.types, dbs.W)
@ -952,7 +954,7 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) {
tx, err := dbs.NewTx(o.Write, indexName, o)
if err != nil {
panic(errors.Wrap(err, "dbs.NewTx transaction errored"))
PanicOn(errors.Wrap(err, "dbs.NewTx transaction errored"))
}
return tx
}
@ -969,7 +971,8 @@ func (ty txtype) String() string {
case boltTxn:
return "bolt"
}
panic(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty)))
PanicOn(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty)))
return ""
}
// fragmentSpecFromRoaringPath takes a path releative to the
@ -1007,7 +1010,7 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64,
// showOps means display the ops log.
func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool, o Txo) (r string) {
paths, err := listFilesUnderDir(idx.path, false, "", true)
panicOn(err)
PanicOn(err)
index := idx.name
r = "allkeys:[\n"
@ -1023,7 +1026,7 @@ func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool, o Txo) (r strin
abspath := idx.path + sep + relpath
s, _, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard, showOps, hashOnly, os.Stdout)
panicOn(err)
PanicOn(err)
//r += fmt.Sprintf("path:'%v' fragment contains:\n") + s
//if s == "" {
//s = "<empty bitmap>"
@ -1044,7 +1047,7 @@ func RoaringFragmentChecksum(path string, index, field, view string, shard uint6
defer func() {
r := recover()
if r != nil {
panic(fmt.Sprintf("caught panic on path='%v', index='%v', field='%v', view='%v', shard='%v': %v",
PanicOn(fmt.Sprintf("caught PanicOn on path='%v', index='%v', field='%v', view='%v', shard='%v': %v",
path, index, field, view, shard, r))
}
}()
@ -1052,7 +1055,7 @@ func RoaringFragmentChecksum(path string, index, field, view string, shard uint6
showOps := false
hashOnly := true
hash, hotbits, err := stringifiedRawRoaringFragment(path, index, field, view, shard, showOps, hashOnly, hasher)
panicOn(err)
PanicOn(err)
fmt.Fprintf(hasher, "%v/%v/%v/%v/%v", index, field, view, shard, hash)
var buf [16]byte
_, _ = hasher.Digest().Read(buf[0:])
@ -1066,14 +1069,14 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard
_ = info
var f *os.File
f, err = os.Open(path)
panicOn(err)
PanicOn(err)
if err != nil {
return
}
var fi os.FileInfo
fi, err = f.Stat()
panicOn(err)
PanicOn(err)
if err != nil {
return
}
@ -1086,9 +1089,9 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard
defer func() {
err := syscall.Munmap(data)
if err != nil {
panic(fmt.Errorf("loadRawRoaringContainer: munmap failed: %v", err))
PanicOn(fmt.Errorf("loadRawRoaringContainer: munmap failed: %v", err))
}
panicOn(f.Close())
PanicOn(f.Close())
}()
// Attach the mmap file to the bitmap.
@ -1115,7 +1118,7 @@ func stringifiedRawRoaringFragment(path string, index, field, view string, shard
}
citer, found := rbm.Containers.Iterator(0)
_ = found // probably gonna use just the Ops log instead, so don't panic if !found.
_ = found // probably gonna use just the Ops log instead, so don't PanicOn if !found.
for citer.Next() {
ckey, ct := citer.Value()
@ -1162,7 +1165,7 @@ func listFilesUnderDir(root string, includeRoot bool, requiredSuffix string, ign
// ignore
} else {
if info == nil {
panic(fmt.Sprintf("info was nil for path = '%v'", path))
PanicOn(fmt.Sprintf("info was nil for path = '%v'", path))
}
if info.IsDir() {
// skip directories.
@ -1205,7 +1208,7 @@ func containerToBytes(ct *roaring.Container) []byte {
ty := roaring.ContainerType(ct)
switch ty {
case roaring.ContainerNil:
panic("nil roaring.Container")
PanicOn("nil roaring.Container")
case roaring.ContainerArray:
return fromArray16(roaring.AsArray(ct))
case roaring.ContainerBitmap:
@ -1213,7 +1216,8 @@ func containerToBytes(ct *roaring.Container) []byte {
case roaring.ContainerRun:
return fromInterval16(roaring.AsRuns(ct))
}
panic(fmt.Sprintf("unknown roaring.Container type '%v'", int(ty)))
PanicOn(fmt.Sprintf("unknown roaring.Container type '%v'", int(ty)))
return nil
}
type pointerContext struct {
@ -1374,7 +1378,9 @@ func (f *TxFactory) greenHasData() (hasData bool, err error) {
case 2:
return f.dbPerShard.HasData(1)
}
panic(fmt.Sprintf("unsupported len(f.types): %v. Must be 1 or 2.", n))
err = fmt.Errorf("unsupported len(f.types): %v. Must be 1 or 2.", n)
PanicOn(err)
return
}
// green2blue is called at the very end of Holder.Open(), so
@ -1461,7 +1467,7 @@ indexloop:
if verifyInsteadOfCopy {
diff := f.shardSetDiff(blueShards, greenShards)
if diff != "" {
return fmt.Errorf("verifyInsteadOfCopy true, blue[%v]=%#v and green[%v]=%#v have different shards for index '%v': '%v'; stack=\n%v", blueDest, blueShards, greenSrc, greenShards, idx.name, diff, stack())
return fmt.Errorf("verifyInsteadOfCopy true, blue[%v]=%#v and green[%v]=%#v have different shards for index '%v': '%v'; stack=\n%v", blueDest, blueShards, greenSrc, greenShards, idx.name, diff, Stack())
}
// can also check against meta data

22
txfactory_internal_test.go
View file

@ -20,6 +20,8 @@ import (
"os"
"testing"
"time"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
func Test_TxFactory_Qcx_query_context(t *testing.T) {
@ -52,7 +54,7 @@ func Test_TxFactory_Qcx_query_context(t *testing.T) {
qcx := idx.holder.txf.NewQcx()
tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: idx, Shard: f.shard})
panicOn(err)
PanicOn(err)
// Set bits on the fragment.
if _, err := f.setBit(tx, 120, 1); err != nil {
@ -72,11 +74,11 @@ func Test_TxFactory_Qcx_query_context(t *testing.T) {
}
finisher(nil) // hit the write tx.Commit path
// commit the change, and verify it is still there
panicOn(qcx.Finish())
PanicOn(qcx.Finish())
qcx.Reset()
tx, finread, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
panicOn(err)
PanicOn(err)
if n := f.mustRow(tx, 120).Count(); n != 2 {
panic(fmt.Sprintf("unexpected count (reopen): %d", n))
} else if n := f.mustRow(tx, 121).Count(); n != 1 {
@ -183,7 +185,7 @@ func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) {
//vv("after close, about to re-open")
// can we re.Open the same holder h? hopefully without a problem.
panicOn(h.Open())
PanicOn(h.Open())
//vv("h.Open() re-open worked; blue_green = '%v'; dump; with PILOSA_STORAGE_BACKEND='%v'", blue_green, os.Getenv("PILOSA_STORAGE_BACKEND"))
//h.DumpAllShards()
@ -200,7 +202,7 @@ func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) {
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = green
h2 := NewHolder(path, cfg)
panicOn(h2.Open())
PanicOn(h2.Open())
testMustHaveBit(t, h2, "i0", "f", rowID, colID)
testMustHaveBit(t, h2, "i1", "f", 100, 200)
@ -212,7 +214,7 @@ func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) {
cfg = mustHolderConfig()
cfg.StorageConfig.Backend = blue
h3 := NewHolder(path, cfg)
panicOn(h3.Open())
PanicOn(h3.Open())
testMustNotHaveBit(t, h3, "i0", "f", rowID, colID)
testMustNotHaveBit(t, h3, "i1", "f", 100, 200)
@ -242,7 +244,7 @@ func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) {
panic("expected error since migration to roaring not supported")
}
} else {
panicOn(err)
PanicOn(err)
}
testMustHaveBit(t, h4, "i0", "f", rowID, colID)
@ -326,7 +328,7 @@ func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) {
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = blue
h3 := NewHolder(path, cfg)
panicOn(h3.Open())
PanicOn(h3.Open())
testMustNotHaveBit(t, h3, "i0", "f", rowID, colID)
testMustNotHaveBit(t, h3, "i1", "f", 100, 200)
@ -350,7 +352,7 @@ func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) {
cfg = mustHolderConfig()
cfg.StorageConfig.Backend = blue_green
h4 := NewHolder(path, cfg)
panicOn(h4.Open())
PanicOn(h4.Open())
testMustHaveBit(t, h4, "i0", "f", rowID, colID)
testMustHaveBit(t, h4, "i1", "f", 100, 200)
@ -362,7 +364,7 @@ func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) {
cfg = mustHolderConfig()
cfg.StorageConfig.Backend = blue
h5 := NewHolder(path, cfg)
panicOn(h5.Open())
PanicOn(h5.Open())
testSetBit(t, h5, "i2", "f", 500, 777)
//vv("after adding a bit to blue, we have:")

11
util.go
View file

@ -28,6 +28,7 @@ import (
"unsafe"
"github.com/pilosa/pilosa/v2/roaring"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
)
@ -147,7 +148,7 @@ func roaringBitmapDiff(a, b *roaring.Bitmap) error {
func dirAsString(path string) (r string) {
r = fmt.Sprintf("dump of directory '%v':\n", path)
files, err := ioutil.ReadDir(path)
panicOn(err)
PanicOn(err)
for _, f := range files {
r += f.Name() + "\n"
}
@ -203,7 +204,7 @@ func fromArray16(a []uint16) []byte {
return []byte{}
}
if len(a) > 4096 {
panic(fmt.Sprintf("cannot put more than 4096 integers into an array container: %v too big", len(a)))
PanicOn(fmt.Sprintf("cannot put more than 4096 integers into an array container: %v too big", len(a)))
}
return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2]
}
@ -222,7 +223,7 @@ func fromInterval16(a []roaring.Interval16) []byte {
return []byte{}
}
if len(a) > 2048 {
panic(fmt.Sprintf("cannot put more than 2048 roaring.Interval16 into a container: %v too big", len(a)))
PanicOn(fmt.Sprintf("cannot put more than 2048 roaring.Interval16 into a container: %v too big", len(a)))
}
return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4]
}
@ -237,7 +238,7 @@ func DiskUse(root string, requiredSuffix string) (tot int, err error) {
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if info == nil {
panic(fmt.Sprintf("info was nil for path = '%v'", path))
PanicOn(fmt.Sprintf("info was nil for path = '%v'", path))
}
if info.IsDir() {
// skip the size of directories themselves, only summing files.
@ -262,7 +263,7 @@ func SubdirLargestDirWithSuffix(rootDir, requiredDirSuffix string) (exists bool,
err = filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
if info == nil {
panic(fmt.Sprintf("info was nil for path = '%v'", path))
PanicOn(fmt.Sprintf("info was nil for path = '%v'", path))
}
if info.IsDir() && strings.HasSuffix(path, requiredDirSuffix) {

5
utils_internal_test.go
View file

@ -24,6 +24,7 @@ import (
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/testhook"
"github.com/pilosa/pilosa/v2/topology"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
)
// utilities used by tests
@ -31,13 +32,13 @@ import (
// mustAddR is a helper for calling roaring.Container.Add() in tests to
// keep the linter happy that we are checking the error.
func mustAddR(changed bool, err error) {
panicOn(err)
PanicOn(err)
}
// mustRemove is a helper for calling Tx.Remove() in tests to
// keep the linter happy that we are checking the error.
func mustRemove(changeCount int, err error) {
panicOn(err)
PanicOn(err)
}
func getTestBitmapAsRawRoaring(bitsToSet ...uint64) []byte {

13
view.go
View file

@ -29,6 +29,7 @@ import (
"github.com/pilosa/pilosa/v2/roaring"
"github.com/pilosa/pilosa/v2/stats"
"github.com/pilosa/pilosa/v2/testhook"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
@ -69,7 +70,7 @@ type view struct {
// newView returns a new instance of View.
func newView(holder *Holder, path, index, field, name string, fieldOptions FieldOptions) *view {
panicOn(validateName(name))
PanicOn(validateName(name))
return &view{
path: path,
@ -115,7 +116,7 @@ func (v *view) addKnownShard(shard uint64) {
atomic.StoreUint32(&v.knownShardsCopied, 0)
}
_, err := v.knownShards.Add(shard)
panicOn(err)
PanicOn(err)
}
// removeKnownShard removes a known shard from v. See the notes on addKnownShard.
@ -458,7 +459,7 @@ func (v *view) setBit(txOrig Tx, rowID, columnID uint64) (changed bool, err erro
tx = v.idx.holder.txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
defer func() {
if err == nil {
panicOn(tx.Commit())
PanicOn(tx.Commit())
} else {
tx.Rollback()
}
@ -480,7 +481,7 @@ func (v *view) clearBit(txOrig Tx, rowID, columnID uint64) (changed bool, err er
tx = v.idx.holder.txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
defer func() {
if err == nil {
panicOn(tx.Commit())
PanicOn(tx.Commit())
} else {
tx.Rollback()
}
@ -520,7 +521,7 @@ func (v *view) setValue(txOrig Tx, columnID uint64, bitDepth uint64, value int64
tx = v.idx.holder.txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
defer func() {
if err == nil {
panicOn(tx.Commit())
PanicOn(tx.Commit())
} else {
tx.Rollback()
}
@ -543,7 +544,7 @@ func (v *view) clearValue(txOrig Tx, columnID uint64, bitDepth uint64, value int
tx = v.idx.holder.txf.NewTx(Txo{Write: writable, Index: v.idx, Fragment: frag, Shard: shard})
defer func() {
if err == nil {
panicOn(tx.Commit())
PanicOn(tx.Commit())
} else {
tx.Rollback()
}

7
view_internal_test.go
View file

@ -19,6 +19,7 @@ import (
"time"
"github.com/pilosa/pilosa/v2/testhook"
. "github.com/pilosa/pilosa/v2/vprint" // nolint:staticcheck
"golang.org/x/sync/errgroup"
)
@ -26,7 +27,7 @@ import (
func mustOpenView(tb testing.TB, index, field, name string) *view {
path, err := testhook.TempDirInDir(tb, *TempDir, "pilosa-view-")
if err != nil {
panic(err)
PanicOn(err)
}
fo := FieldOptions{
@ -47,12 +48,12 @@ func mustOpenView(tb testing.TB, index, field, name string) *view {
testhook.Cleanup(tb, func() {
h.Close()
})
panicOn(err)
PanicOn(err)
v := newView(h, path, index, field, name, fo)
v.idx = idx
if err := v.openEmpty(); err != nil {
panic(err)
PanicOn(err)
}
v.rowAttrStore = &memAttrStore{
store: make(map[uint64]map[string]interface{}),

67
vprint.go → vprint/vprint.go
View file

@ -1,28 +1,18 @@
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
// Copyright 2020 Pilosa Corp.
//
// MIT License
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// http://www.apache.org/licenses/LICENSE-2.0
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
package vprint
import (
"fmt"
@ -41,21 +31,28 @@ const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
_ = DirExists
_ = FileExists
_ = Caller
_ = Stack
_ = RFC3339MsecTz0
_ = RFC3339UsecTz0
_ = AlwaysPrintf
_ = FileSize
}
func PanicOn(err error) {
if err != nil {
panic(err)
func PanicOn(err interface{}) {
switch v := err.(type) {
case string:
if v != "" {
panic(v)
}
default: // error
if v != nil {
panic(v)
}
}
}
@ -78,8 +75,8 @@ var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("\n%s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
_, _ = Printf("\n%s %s ", FileLine(3), ts())
_, _ = Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
@ -108,7 +105,7 @@ func FileLine(depth int) string {
return s
}
func stack() string {
func Stack() string {
return string(debug.Stack())
}

176
vprint_test.go
View file

@ -1,176 +0,0 @@
// home: https://github.com/glycerine/vprint
// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved.
// License: MIT
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package pilosa_test
import (
"fmt"
"io"
"os"
"path"
"runtime"
"runtime/debug"
"sync"
"time"
)
const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00"
const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00"
// for tons of debug output
var VerboseVerbose bool = false
// convience functions for . import
var pp = PP
var vv = VV
var panicOn = PanicOn
func init() {
// keeper linter happy
_ = pp
_ = vv
_ = DirExists
}
func PanicOn(err error) {
if err != nil {
panic(err)
}
}
func PP(format string, a ...interface{}) {
if VerboseVerbose {
TSPrintf(format, a...)
}
}
func VV(format string, a ...interface{}) {
TSPrintf(format, a...)
}
func AlwaysPrintf(format string, a ...interface{}) {
TSPrintf(format, a...)
}
var tsPrintfMut sync.Mutex
// time-stamped printf
func TSPrintf(format string, a ...interface{}) {
tsPrintfMut.Lock()
Printf("\n%s %s ", FileLine(3), ts())
Printf(format+"\n", a...)
tsPrintfMut.Unlock()
}
// get timestamp for logging purposes
func ts() string {
return time.Now().Format(RFC3339UsecTz0)
}
// so we can multi write easily, use our own printf
var OurStdout io.Writer = os.Stdout
// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func Printf(format string, a ...interface{}) (n int, err error) {
return fmt.Fprintf(OurStdout, format, a...)
}
func FileLine(depth int) string {
_, fileName, fileLine, ok := runtime.Caller(depth)
var s string
if ok {
s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine)
} else {
s = ""
}
return s
}
func stack() string {
return string(debug.Stack())
}
// happy linter
var _ = stack
var _ = FileExists
var _ = FileSize
var _ = AlwaysPrintf
var _ = RFC3339MsecTz0
var _ = Caller
func FileExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return false
}
return true
}
func DirExists(name string) bool {
fi, err := os.Stat(name)
if err != nil {
return false
}
if fi.IsDir() {
return true
}
return false
}
func FileSize(name string) (int64, error) {
fi, err := os.Stat(name)
if err != nil {
return -1, err
}
return fi.Size(), nil
}
// Caller returns the name of the calling function.
func Caller(upStack int) string {
// elide ourself and runtime.Callers
target := upStack + 2
pc := make([]uintptr, target+2)
n := runtime.Callers(0, pc)
f := runtime.Frame{Function: "unknown"}
if n > 0 {
frames := runtime.CallersFrames(pc[:n])
for i := 0; i <= target; i++ {
contender, more := frames.Next()
if i == target {
f = contender
}
if !more {
break
}
}
}
return f.Function
}