diff --git a/api.go b/api.go index ca3506a0e..7bcc0dda7 100644 --- a/api.go +++ b/api.go @@ -1510,9 +1510,7 @@ func addClearToImportOptions(opts []ImportOption) []ImportOption { return append(opts, OptImportOptionsClear(true)) } -// Import avoids re-writing a bajillion tests to be transaction-aware by allowing a nil pQcx. -// It is convenient for some tests, particularly those in loops, to pass a nil qcx and -// treat the Import as having been commited when we return without error. We make it so. +// Import does the top-level importing. func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts ...ImportOption) (err error) { if req.Clear { opts = addClearToImportOptions(opts) @@ -1648,8 +1646,8 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, return errors.Wrap(err, "committing") } -// ImportValue avoids re-writing a bajillion tests by allowing a nil pQcx. -// Then we will commit before returning. +// ImportValue is a wrapper around the common code in ImportValueWithTx, which +// currently just translates req.Clear into a clear ImportOption. func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error { if req.Clear { opts = addClearToImportOptions(opts) @@ -2654,9 +2652,7 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64 if err != nil { return err } - //need to find the path to the db - //will not work on blue green - db := dbs.W[0] + db := dbs.W finalPath := db.Path() + "/data" tempPath := finalPath + ".tmp" o, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) diff --git a/barrier.go b/barrier.go deleted file mode 100644 index b150dc486..000000000 --- a/barrier.go +++ /dev/null @@ -1,219 +0,0 @@ -// home https://github.com/glycerine/lmdb-go -// Copyright (c) 2020, the lmdb-go authors -// Copyright (c) 2015, Bryan Matsuo -// All rights reserved. - -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: - -// Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. - -// Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. - -// Neither the name of the author nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. - -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package pilosa - -import ( - "github.com/glycerine/idem" -) - -// Barrier allows us to temporarily halt all readers, so that -// a writer can commit alone and thus compact the db. -// The Barrier starts unblocked, alllowing passage to any -// caller of WaitAtGate(). -type Barrier struct { - wait chan *appointment // send upon entering the waiting room. - halt *idem.Halter - blockReqCh chan *blockReq - unblockCh chan *unblock -} - -type blockReq struct { - count int - done chan struct{} -} - -func newBlockReq(count int) *blockReq { - return &blockReq{ - count: count, - done: make(chan struct{}), - } -} - -type appointment struct { - id int - done chan struct{} -} - -func newAppointment(id int) *appointment { - return &appointment{ - id: id, - done: make(chan struct{}), - } -} - -// NewBarrier is either open, allowing immediate passage, -// or blocked, halting all callers at WaitAtGate() -// until the barrier is opened. By default it is open. -// -// Barrier.Close() must be called when the barrier -// is no longer needed to avoid a goroutine leak. -func NewBarrier() (b *Barrier) { - b = &Barrier{ - wait: make(chan *appointment), // waiters indicate they are waiting for the gate by sending here. - halt: idem.NewHalter(), - blockReqCh: make(chan *blockReq), - unblockCh: make(chan *unblock), - } - go func() { - defer b.halt.Done.Close() - - var waitlist []*appointment - var curBlockReq *blockReq - - for { - select { - case br := <-b.blockReqCh: - if br.count == 0 { - close(br.done) - continue - } - if curBlockReq == nil { - // good, changing state from open to closed barrier. - } else { - panic("got 2nd block request atop of first") - } - curBlockReq = br - //vv("barrier: request to block for %v waiters", br.count) - if len(waitlist) != 0 { - panic("had waiters when we were open, internal/client bug") - } - case appt := <-b.wait: - //vv("barrier.wait sees appt = '%#v' and curBlockReq = '%#v'", appt, curBlockReq) - if curBlockReq == nil { - close(appt.done) - continue - } - waitlist = append(waitlist, appt) - n := len(waitlist) - th := curBlockReq.count - if th < 0 { - // infinite waiters. we block everybody until we - // see an unblock request. - continue - } - if n >= th { - close(curBlockReq.done) - curBlockReq = nil - } - case ub := <-b.unblockCh: - for _, appt := range waitlist { - close(appt.done) - } - waitlist = nil - curBlockReq = nil - close(ub.done) - case <-b.halt.ReqStop.Chan: - return - } - } - }() - return -} - -// WaitAtGate will return immediately -// if the barrier is unblocked. Otherwise -// it will not return until another -// goroutine unblocks the barrier. -func (b *Barrier) WaitAtGate(id int) { - appt := newAppointment(id) - select { - case b.wait <- appt: - select { - case <-appt.done: - case <-b.halt.ReqStop.Chan: - } - case <-b.halt.ReqStop.Chan: - } -} - -// Close should be called to stop the -// barrier's background goroutine when -// you are done using the barrier. -func (b *Barrier) Close() { - b.halt.ReqStop.Close() - <-b.halt.Done.Chan -} - -type unblock struct { - done chan struct{} -} - -func newUnblock() *unblock { - return &unblock{ - done: make(chan struct{}), - } -} - -// Unblock lets all waiting goroutines resume execution. -func (b *Barrier) UnblockReaders() { - ub := newUnblock() - select { - case b.unblockCh <- ub: - select { - case <-ub.done: - case <-b.halt.ReqStop.Chan: - } - case <-b.halt.ReqStop.Chan: - } -} - -// BlockUntil is called with a count, the -// number of waiters required to be present and waiting -// at the gate before call returns. -// A count of < 0 will return immediately and raise -// the barrier to any number of arriving readers. -// A count of 0 is a no-op. -// -// Otherwise we raise the barrier -// and wait until we have seen count other goroutines waiting -// on it. -// -// We return without releasing the waiters. Call -// Open when you want them to resume. -func (b *Barrier) BlockUntil(count int) { - if count == 0 { - return - } - req := newBlockReq(count) - b.blockReqCh <- req - if count > 0 { - <-req.done - } -} - -// BlockAllReadersNoWait raises the barrier to -// an infinite number of waiters and returns immediately -// to the caller. -func (b *Barrier) BlockAllReadersNoWait() { - req := newBlockReq(-1) // -1 means block any number of readers. - b.blockReqCh <- req - // don't wait. <-req.done -} diff --git a/barrier_test.go b/barrier_test.go deleted file mode 100644 index 70fd4d311..000000000 --- a/barrier_test.go +++ /dev/null @@ -1,90 +0,0 @@ -// home https://github.com/glycerine/lmdb-go -// Copyright (c) 2020, the lmdb-go authors -// Copyright (c) 2015, Bryan Matsuo -// All rights reserved. - -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: - -// Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. - -// Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. - -// Neither the name of the author nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. - -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package pilosa - -import ( - "fmt" - "sync/atomic" - "testing" - "time" -) - -func TestBarrierHolds(t *testing.T) { - b := NewBarrier() - defer b.Close() - - released := int64(0) - - waiter := func(i int) { - b.WaitAtGate(i) - //vv("goro %v is released", i) - atomic.AddInt64(&released, 1) - } - - for i := 0; i < 3; i++ { - go waiter(i) - } - time.Sleep(time.Second) - r := atomic.SwapInt64(&released, 0) - if r != 3 { - panic("open barrier held back goro") - } - //vv("good: barrier started open") - - //seenAll := make(chan bool) - b.BlockAllReadersNoWait() - for i := 0; i < 3; i++ { - go waiter(i) - } - - time.Sleep(time.Second) - r = atomic.SwapInt64(&released, 0) - if r != 0 { - panic("bad: barrier did not hold back goro") - } - //vv("good: barrier of 4 did not release on 3") - go waiter(4) - - time.Sleep(time.Second) - r = atomic.SwapInt64(&released, 0) - if r != 0 { - panic(fmt.Sprintf("bad: barrier did not hold back goro, should wait for unblock. r = %v", r)) - } - - b.UnblockReaders() - - time.Sleep(time.Second) - r = atomic.SwapInt64(&released, 0) - if r != 4 { - panic("bad: unblock should have released 4 goro") - } - -} diff --git a/bluegreentx.go b/bluegreentx.go deleted file mode 100644 index 07c716437..000000000 --- a/bluegreentx.go +++ /dev/null @@ -1,1028 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "fmt" - "io" - "reflect" - "sync" - - "github.com/molecula/featurebase/v2/roaring" - txkey "github.com/molecula/featurebase/v2/short_txkey" - . "github.com/molecula/featurebase/v2/vprint" -) - -// blueGreenTx runs two Tx together and notices differences in their output. -// By convention, the 'b' Tx is the output that is returned to caller. -// -// Warning: DATA RACES are expected if RoaringTx is one side of the Tx pair. -// The checkDatabase() call will do reads of the fragments at Commit/Rollback, -// while the snapshotqueue may be doing writes. -// -// Do not run with go test -race and expect it to be race free with RoaringTx -// on one arm. -// -// Note: using the dbshard.go DBShard.mut RWMutex to begin and end -// both the A and B transactions atomically, we support a single importer and -// lots of readers running under blue-green transactions. Two writers a.k.a. two -// github ingests at once will deadlock eventually, but I think that may be asking -// for more than we want to test under blue-green, as it would require a bunch of -// test-only internal executor logic that could mess with the production path. -// So, for now, a limitation on blue green tests is that they be single -// writer/single importer going at once. -// -type blueGreenTx struct { - a Tx - b Tx // b's output is returned - - o Txo - as string - bs string - - types []txtype - hasRoaring bool - - // roaring will not create as many Tx (they are - // psuedo Tx anyway), espcially when deleting - // files. Return the non-roaring Sn if - // possible, by referencing useSnA. - useSnA bool - - idx *Index - - checker blueGreenChecker - mu sync.Mutex - rollbackOrCommitDone bool - - txf *TxFactory - - short bool // short Dump or long - - FullDump bool // else quieter, don't attemp Dump() if false. -} - -// blueGreenRegistry is used to force checking of (read) transactions -// before writes happen, if roaring is on one of the A/B branches. -// Because roaring won't have an MVCC view of the world. Writes to -// roaring will show up, while writes to the DB won't show up on -// readTx that have already started. -type blueGreenRegistry struct { - mu sync.Mutex - m map[int64]*blueGreenTx - types []txtype - hasRoaring bool -} - -// if we have raoring in the mix we cannot expect reads -// to match up, but otherwise do. -func newBlueGreenReg(types []txtype) *blueGreenRegistry { - - hasRoaring := false - if types[0] == roaringTxn || types[1] == roaringTxn { - hasRoaring = true - } - return &blueGreenRegistry{ - m: make(map[int64]*blueGreenTx), - types: types, - hasRoaring: hasRoaring, - } -} - -// add remembers the tx so we can check that -// all tx were finished before Close(). -func (b *blueGreenRegistry) add(c *blueGreenTx) { - b.mu.Lock() - defer b.mu.Unlock() - if c.useSnA { - b.m[c.a.Sn()] = c - } else { - b.m[c.b.Sn()] = c - } -} - -func (b *blueGreenRegistry) finishedTx(tx *blueGreenTx) { - b.mu.Lock() - defer b.mu.Unlock() - sn := tx.Sn() - delete(b.m, sn) - //vv("blueGreenRegistry deleted _sn_ %v", sn) - - // Note that a tx.o.dbs.Cleanup(tx) call should not be needed, - // because the individual tx will call cleanup themselves. -} - -func (b *blueGreenRegistry) Close() { - b.mu.Lock() - defer b.mu.Unlock() - if len(b.m) > 0 { - PanicOn(fmt.Sprintf("still have open/unchecked blueGreenTx: '%#v'", b.m)) - //AlwaysPrintf("still have unchecked blueGreenTx: '%#v'", b.m) - } -} - -func (txf *TxFactory) newBlueGreenTx(a, b Tx, idx *Index, o Txo) *blueGreenTx { - as := a.Type() - bs := b.Type() - c := &blueGreenTx{a: a, - b: b, - idx: idx, - as: as, - bs: bs, - txf: txf, - types: txf.types, - hasRoaring: txf.blueGreenReg.hasRoaring, - short: true, - } - - if c.types[1] == roaringTxn { - c.useSnA = true - } - //vv("newBlueGreenTx with a.sn=%v with o.Shard=%v", c.Sn(), int(o.Shard)) - - c.checker.c = c - c.o = o - txf.blueGreenReg.add(c) - return c -} - -var _ Tx = (*blueGreenTx)(nil) - -func (c *blueGreenTx) Type() string { - return c.a.Type() + "_" + c.b.Type() -} - -var blueGreenTxDumpMut sync.Mutex - -func (c *blueGreenTx) Dump(short bool, shard uint64) { - - if !c.FullDump { - return - } - - blueGreenTxDumpMut.Lock() - defer blueGreenTxDumpMut.Unlock() - fmt.Printf("%v blueGreenTx.Dump ============== \n", FileLine(2)) - fmt.Printf("A(%v) Dump:\n", c.as) - c.a.Dump(short, shard) - fmt.Printf("B(%v) Dump:\n", c.bs) - c.b.Dump(short, shard) - - if !short { - fmt.Printf("dbPerShard.DumpAll(): idx=%p\n", c.idx) - c.idx.holder.txf.dbPerShard.DumpAll() - } -} - -func (c *blueGreenTx) Readonly() bool { - a := c.a.Readonly() - b := c.b.Readonly() - if a != b { - PanicOn(fmt.Sprintf("Readonly difference, a=%v, but b =%v", a, b)) - } - return b -} - -// for now we just return B's list, since this is involved in -// holder Open which can happen before any blue-green is done; -// in fact this is instrumental in setting up the sync from -// green to blue. -func (c *blueGreenTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvB []txkey.FieldView, errB error) { - fvB, errB = c.b.GetSortedFieldViewList(idx, shard) - return -} - -func (c *blueGreenTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { - c.checker.see(index, field, view, shard) - // can't really do simultaneous iteration on A and B, so punt and - // just give back B. - return c.b.NewTxIterator(index, field, view, shard) -} - -func (c *blueGreenTx) Pointer() string { - return fmt.Sprintf("%p", c) -} - -func (c *blueGreenTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - c.checker.see(index, field, view, shard) - c.a.IncrementOpN(index, field, view, shard, changedN) - c.b.IncrementOpN(index, field, view, shard, changedN) -} - -// compareTxState is called for the first Commit or Rollback a blueGreenTx sees. -func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) { - if c.o.blueGreenOff { - return - } - here := fmt.Sprintf("%v/%v/%v/%v", index, field, view, shard) - //vv("compareTxState here = '%v', _sn_ %v gid=%v", here, c.Sn(), curGID()) - aIter, aFound, aErr := c.a.ContainerIterator(index, field, view, shard, 0) - bIter, bFound, bErr := c.b.ContainerIterator(index, field, view, shard, 0) - if aErr == nil || aIter != nil { - defer aIter.Close() - } - if bErr == nil || bIter != nil { - defer bIter.Close() - } - - if aFound != bFound { - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) ContainerIterator had aFound=%v, but B(%v) had bFound=%v; at '%v'", here, c.as, aFound, c.bs, bFound, Stack())) - } - - if aErr != nil || bErr != nil { - if aErr != nil && bErr != nil { - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) reported err '%v'; B(%v) reported err '%v' at %v", here, c.as, aErr, c.bs, bErr, Stack())) - } - if aErr != nil { - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) reported err %v at %v; but B(%v) did not", here, c.as, aErr, c.bs, Stack())) - } - if bErr != nil { - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, Stack())) - } - } - for aIter.Next() { - aKey, aValue := aIter.Value() - - if !bIter.Next() { - AlwaysPrintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, dump to follow, Stack=\n %v\n\n and here is dump:", here, c.as, aKey, c.bs, Stack()) - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, Stack())) - } - bKey, bValue := bIter.Value() - if bKey != aKey { - AlwaysPrintf("problem in caller %v", Caller(2)) - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, Stack())) - } - if err := aValue.BitwiseCompare(bValue); err != nil { - c.Dump(c.short, shard) - //vv("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at Stack=%v", here, aKey, err, c.as, c.bs, Stack()) - PanicOn(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at Stack=%v", here, aKey, err, c.as, c.bs, Stack())) - } - //vv("successfully matched aKey(%v)='%v' and bKey(%v)='%v'", c.as, aKey, c.bs, bKey) - } - // end checking everything in A, but does B have more? - if bIter.Next() { - AlwaysPrintf("bIter has more than it should. problem in caller %v. _sn_ %v", Caller(2), c.Sn()) - c.Dump(c.short, shard) - bKey, _ := bIter.Value() - PanicOn(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, (a.sn=%v) (b.sn=%v) at %v", here, c.bs, bKey, c.as, c.a.Sn(), c.b.Sn(), Stack())) - } - //vv("done without problem. compareTxState here = '%v', _sn_ %v gid=%v", here, c.Sn(), curGID()) -} - -func (c *blueGreenTx) checkDatabase() { - if c.o.blueGreenOff { - return - } - if c.hasRoaring && !c.o.Write { - // With roaring on one arm, we only check the we are A/B - // consistent after every write. - // - // Ideally reads can only see that consitent state, and don't need - // to be checked themselves-- but we do try if both A and B - // are transactional. Sketch of proof by induction that - // write checking should, theoretically, suffice: - // Starting with zero data, if we have agreement in both A/B - // database state after each write, then - // because there is only ever a single - // writer (for LMDB/RBF), we should always have the same - // data state between A and B as long as every prior - // A/B check of the serialized writes suceeded. - // - // This avoids a key problem we discovered when A/B checking reads - // with roaring on one arm. - // The MVCC of the transactional engines means that reads that - // start before a write commit will look very different - // when comparing to roaring's non-transactional state. - return - } - - c.checker.mu.Lock() - defer c.checker.mu.Unlock() - if c.checker.checkDone { - return // idemopotent. checkDatabase can be called twice. Only the first does the checks. - } - c.checker.checkDone = true - - for index, fields := range c.checker.seen() { - for field, views := range fields { - for view, shards := range views { - for shard := range shards { - c.compareTxState(index, field, view, shard) - } - } - } - } -} - -func (c *blueGreenTx) IsDone() bool { - return c.b.IsDone() -} - -func (c *blueGreenTx) Rollback() { - c.mu.Lock() - defer c.mu.Unlock() - if c.rollbackOrCommitDone { - return // avoid using discarded tx for Dump, which will PanicOn. - } - c.rollbackOrCommitDone = true - - if c.o.Write { - c.checkDatabase() - } - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - //vv("blueGreenTx.Rollback() about to call (%v) a.Rollback()", c.as) - c.a.Rollback() - //vv("blueGreenTx.Rollback() about to call (%v) b.Rollback()", c.bs) - c.b.Rollback() - //vv("blueGreenTx.Rollback() done. bgtx p=%p", c) - - c.txf.blueGreenReg.finishedTx(c) -} - -func (c *blueGreenTx) Commit() error { - c.mu.Lock() - defer c.mu.Unlock() - - if c.rollbackOrCommitDone { - return nil - } - //vv("blueGreenTx.Commit() called. bgtx p=%p", c) - c.rollbackOrCommitDone = true - if c.o.Write { - if !c.o.blueGreenOff { - c.checkDatabase() - } - } - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - errA := c.a.Commit() - _ = errA - errB := c.b.Commit() - - compareErrors(errA, errB) - c.txf.blueGreenReg.finishedTx(c) - return errB -} - -func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.RoaringBitmap(index, field, view, shard) - _, _ = a, errA - b, errB := c.b.RoaringBitmap(index, field, view, shard) - if !c.o.blueGreenOff { - compareErrors(errA, errB) - - slcA := a.Slice() - slcB := b.Slice() - if !reflect.DeepEqual(slcA, slcB) { - PanicOn("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!") - } - } - return b, errB -} - -func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.Container(index, field, view, shard, key) - b, errB := c.b.Container(index, field, view, shard, key) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - err = a.BitwiseCompare(b) - PanicOn(err) - } - return b, errB -} - -func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - errA := c.a.PutContainer(index, field, view, shard, key, rc) - errB := c.b.PutContainer(index, field, view, shard, key, rc) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return errB -} - -func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { - c.checker.see(index, field, view, shard) - - // these are the first port of call for debugging, so we leave them in. - // ================== begin save comments. - //c.checkDatabase() - ////vv("got past database check at TOP of ImportRoaringBits") - //c.Dump(c.short, shard) - ////vv("done with top dump; clear=%v", clear) - // ================== end save comments. - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - - // remember where the iterator started, so we can replay it a second time. - rit2 := rit.Clone() - PanicOn(err) - - changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) - changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize, data) - - if !c.o.blueGreenOff { - - if len(data) == 0 { - // okay to check! otherwise we are in the fragment.fillFragmentFromArchive - // case where we know that RoaringTx.ImportRoaringBits changed and rowSet will - // be inaccurate. - if changedA != changedB { - PanicOn(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB)) - } - if len(rowSetA) != len(rowSetB) { - PanicOn(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB)) - } - for k, va := range rowSetA { - vb, ok := rowSetB[k] - if !ok { - PanicOn(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB)) - } - if va != vb { - PanicOn(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb)) - } - } - } - compareErrors(errA, errB) - c.checkDatabase() - } - return changedB, rowSetB, errB -} - -func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - errA := c.a.RemoveContainer(index, field, view, shard, key) - errB := c.b.RemoveContainer(index, field, view, shard, key) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return errB -} - -func (c *blueGreenTx) UseRowCache() bool { - // avoid cross-talk between our two implementations - // by never allowing either to use the row cache. - return false -} - -var _ = (&blueGreenTx{}).isIn // happy linter - -func (c *blueGreenTx) isIn(index, field, view string, shard uint64, ckey uint64) (r []bool) { - r = make([]bool, 2) - inA, errA := c.a.Contains(index, field, view, shard, ckey) - PanicOn(errA) - inB, errB := c.b.Contains(index, field, view, shard, ckey) - PanicOn(errB) - r[0] = inA - r[1] = inB - return -} - -func (c *blueGreenTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - c.checker.see(index, field, view, shard) - //vv("blueGreenTx) Add(index=%v, field=%v, view=%v, shard=%v", index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Add() PanicOn '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, Stack()) - PanicOn(r) - } - }() - - // must copy a before calling Add(), since RoaringTx.Add() uses roaring.DirectAddN() - // which modifies the input array a. - a2 := make([]uint64, len(a)) - copy(a2, a) - - ach, errA := c.a.Add(index, field, view, shard, a...) - _, _ = ach, errA - - bch, errB := c.b.Add(index, field, view, shard, a2...) - - if !c.o.blueGreenOff { - - if ach != bch { - PanicOn(fmt.Sprintf("Add() difference, ach=%v, but bch=%v; errA='%v'; errB='%v'", ach, bch, errA, errB)) - } - compareErrors(errA, errB) - } - - return bch, errB -} - -func compareErrors(errA, errB error) { - switch { - case errA == nil && errB == nil: - // OK - case errA == nil: - PanicOn(fmt.Sprintf("errA is nil, but errB = %#v", errB)) - case errB == nil: - PanicOn(fmt.Sprintf("errB is nil, but errA = %#v", errA)) - default: - ae := errA.Error() - be := errB.Error() - if ae != be { - PanicOn(fmt.Sprintf("errA is '%v', but errB is '%v'", ae, be)) - } - } -} - -func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - ach, errA := c.a.Remove(index, field, view, shard, a...) - _, _ = ach, errA - bch, errB := c.b.Remove(index, field, view, shard, a...) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return bch, errB -} - -func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - ax, errA := c.a.Contains(index, field, view, shard, key) - _, _ = ax, errA - bx, errB := c.b.Contains(index, field, view, shard, key) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return bx, errB -} - -func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - - ait, afound, errA := c.a.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) - _, _, _ = ait, afound, errA - - bit, bfound, errB := c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - - if errB != nil { - // RoaringTx can return an iterator and an error, so be sure Close it we have it. - if ait != nil { - ait.Close() - } - if bit != nil { - bit.Close() - } - return nil, bfound, errB - } - if errA != nil { - // RoaringTx can return an iterator and an error, so be sure Close it we have it. - if ait != nil { - ait.Close() - } - } - - // INVAR: errA == errB == nil - bgi := NewBlueGreenIterator(c, ait, bit) - return bgi, bfound, errB -} - -func (tx *blueGreenTx) GetFieldSizeBytes(index, field string) (uint64, error) { - return 0, nil -} - -func NewBlueGreenIterator(tx *blueGreenTx, ait, bit roaring.ContainerIterator) *blueGreenIterator { - return &blueGreenIterator{ - tx: tx, - as: tx.as, - bs: tx.bs, - ait: ait, - bit: bit, - } -} - -type blueGreenIterator struct { - tx *blueGreenTx - as string - bs string - - ait roaring.ContainerIterator - bit roaring.ContainerIterator -} - -func (bgi *blueGreenIterator) Next() bool { - na := bgi.ait.Next() - nb := bgi.bit.Next() - if na != nb { - PanicOn(fmt.Sprintf("na=%v(%v) != nb(%v)=%v", na, bgi.as, bgi.bs, nb)) - } - return nb -} - -func (bgi *blueGreenIterator) Value() (uint64, *roaring.Container) { - ka, ca := bgi.ait.Value() - kb, cb := bgi.bit.Value() - - if !bgi.tx.o.blueGreenOff { - if ka != kb { - PanicOn(fmt.Sprintf("ka=%v != kb=%v", ka, kb)) - } - err := ca.BitwiseCompare(cb) - PanicOn(err) - } - return kb, cb -} -func (bgi *blueGreenIterator) Close() { - bgi.ait.Close() - bgi.bit.Close() -} - -// ForEach is read-only on the database, and so we only pass through to B. -// Avoids the side-effects of calling fn too many times, which can cause serious false alarms. -func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.ForEach(index, field, view, shard, fn) - -} - -// ForEachRange cannot change the database, and we also can't control -// the side effects of the fn() calls. So we only pass through to B, not A. -// No checker.see() is needed as well, because we are read-only. -func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { - - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - - // calling fn will have side effects; can only call it the right number of times. - // so can't do this. - // errA := c.a.ForEachRange(index, field, view, shard, start, end, fn) - return c.b.ForEachRange(index, field, view, shard, start, end, fn) -} - -func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.Count(index, field, view, shard) - _, _ = a, errA - b, errB := c.b.Count(index, field, view, shard) - _, _ = b, errB - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return b, errB -} - -func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.Max(index, field, view, shard) - _, _ = a, errA - b, errB := c.b.Max(index, field, view, shard) - _, _ = b, errB - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return b, errB -} - -func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - amin, afound, errA := c.a.Min(index, field, view, shard) - _, _, _ = amin, afound, errA - bmin, bfound, errB := c.b.Min(index, field, view, shard) - _, _, _ = bmin, bfound, errB - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return bmin, bfound, errB -} - -func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - errA := c.a.UnionInPlace(index, field, view, shard, others...) - errB := c.b.UnionInPlace(index, field, view, shard, others...) - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return errB -} - -func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - c.Dump(c.short, shard) - AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.CountRange(index, field, view, shard, start, end) - b, errB := c.b.CountRange(index, field, view, shard, start, end) - - if !c.o.blueGreenOff { - if a != b { - PanicOn(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b)) - } - - compareErrors(errA, errB) - } - return b, errB -} - -func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see OffsetRange() on _sn_ %v, PanicOn '%v' at '%v'", c.Sn(), r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.OffsetRange(index, field, view, shard, offset, start, end) - b, errB := c.b.OffsetRange(index, field, view, shard, offset, start, end) - - if !c.o.blueGreenOff { - - err = roaringBitmapDiff(a, b) - if err != nil { - c.Dump(false, shard) - PanicOn(fmt.Errorf("on _sn_ %v OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) err: %v", c.Sn(), index, field, view, int(shard), offset, start, end, err)) - } - compareErrors(errA, errB) - } - - return b, errB -} - -func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - c.Dump(c.short, shard) - AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - - rcA, szA, errA := c.a.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) - rcB, szB, errB := c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - - // We are seeing Roaring vs Badger size differences on - // server/ test TestClusterResize_AddNode/ContinuousShards, - // so turn off the szA vs szB checks and MutliReaderB use. But keep them if we want to - // check RBF vs Badger for byte-for-byte compatiblity (we - // suspect the ops log or optimized bitmaps are accounting for the difference). - sizeMustMatch := false // !c.hasRoaring - if c.o.blueGreenOff { - sizeMustMatch = false - } - if sizeMustMatch { - if szA != szB { - PanicOn(fmt.Sprintf("szA(%v) = %v, but szB(%v) = %v; fragmentPathForRoaring='%v'", c.as, szA, c.bs, szB, fragmentPathForRoaring)) - } - return &MultiReaderB{a: rcA, b: rcB}, szB, errB - } else { - // one db won't get data if we do - //return &MultiReaderB{a: rcA, b: rcB, allowSizeVariation: true}, szB, errB - _, _ = szA, errA - rcA.Close() - return rcB, szB, errB - } -} - -func (c *blueGreenTx) Group() *TxGroup { - return c.b.Group() -} - -func (c *blueGreenTx) Options() Txo { - return c.b.Options() -} - -// Sn retreives the serial number of the Tx. -func (c *blueGreenTx) Sn() int64 { - asn := c.a.Sn() - bsn := c.b.Sn() - - if c.useSnA { - return asn - } - return bsn -} - -func (c *blueGreenTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { - return GenericApplyFilter(c, index, field, view, shard, ckey, filter) -} - -// MultiReaderB is returned by RoaringBitmapReader. It verifies -// that identical byte streams are read from its two members. -type MultiReaderB struct { - a io.ReadCloser - b io.ReadCloser - - allowSizeVariation bool -} - -// Read implements the standard io.Reader method. It panics -// if "a" and "b" have even one byte different in their reads. -func (m *MultiReaderB) Read(p []byte) (nB int, errB error) { - nB, errB = m.b.Read(p) - p2 := make([]byte, nB) - - // read (and discard after comparing for equality) the exact same amount from A. - - // ReadAtLeast reads from r into buf until it has read at least - // min bytes. It returns the number of bytes copied and an error - // if fewer bytes were read. The error is EOF only if no bytes - // were read. If an EOF happens after reading fewer than min bytes, - // ReadAtLeast returns ErrUnexpectedEOF. If min is greater than - // the length of buf, ReadAtLeast returns ErrShortBuffer. On - // return, n >= min if and only if err == nil. If r returns - // an error having read at least min bytes, the error is dropped. - nA, errA := io.ReadAtLeast(m.a, p2, nB) - - if !m.allowSizeVariation { - if errA == io.ErrUnexpectedEOF { - PanicOn(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA)) - } - if nA != nB { - PanicOn(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA)) - } - cmp := bytes.Compare(p[:nB], p2[:nB]) - if cmp != 0 { - PanicOn(fmt.Sprintf("MultiReaderB reads p and p2 (cmp= %v) differed.", cmp)) - } - } - return -} - -func (m *MultiReaderB) Close() error { - m.a.Close() - return m.b.Close() -} - -// blueGreenChecker is used -type blueGreenChecker struct { - visited map[string]map[string]map[string]map[uint64]struct{} - - c *blueGreenTx - - // lock mu when using visited. - // otherwise concurrent map writes on TestAPI_Import/RowIDColumnKey - mu sync.Mutex - - checkDone bool -} - -// see would mark a thing as seen. -func (b *blueGreenChecker) see(index, field, view string, shard uint64) { - // keep this next Printf. Useful to see the sequence of Tx operations. - //fmt.Printf("blueGreenTx.%v on index='%v' shard=%v\n", Caller(1), index, shard) - - if !b.c.o.Write { - return - } - - b.mu.Lock() - defer b.mu.Unlock() - - if b.visited == nil { - b.visited = make(map[string]map[string]map[string]map[uint64]struct{}) - } - var visitedIdx map[string]map[string]map[uint64]struct{} - var visitedField map[string]map[uint64]struct{} - var visitedView map[uint64]struct{} - - if visitedIdx = b.visited[index]; visitedIdx == nil { - visitedIdx = make(map[string]map[string]map[uint64]struct{}) - b.visited[index] = visitedIdx - } - if visitedField = visitedIdx[field]; visitedField == nil { - visitedField = make(map[string]map[uint64]struct{}) - visitedIdx[field] = visitedField - } - if visitedView = visitedField[view]; visitedView == nil { - visitedView = make(map[uint64]struct{}) - visitedField[view] = visitedView - } - visitedView[shard] = struct{}{} -} - -// seen reports the things it has seen, exactly once so -// that Rollback can be called after Commit without repeating -// the check. -func (b *blueGreenChecker) seen() map[string]map[string]map[string]map[uint64]struct{} { - return b.visited -} diff --git a/bluegreentx_test.go b/bluegreentx_test.go deleted file mode 100644 index 3a365800b..000000000 --- a/bluegreentx_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "context" - "io" - "io/ioutil" - "os" - "strings" - "testing" - - cryrand "crypto/rand" - - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck -) - -var _ = context.Background -var _ = os.Open -var _ = strings.Split - -func TestMultiReaderB(t *testing.T) { - // MultiReaderB should read identical chunks of bytes from both its "a" and "b" - // member io.Readers, else it should panic. This should hold for - // varying sizes of inputs. - - for n := 1 << 5; n < (1 << 18); n = n*2 - 13 { - src := io.LimitReader(cryrand.Reader, int64(n)) - - a := make([]byte, n) - nr := 0 - for nr < n { - na, err := src.Read(a) - PanicOn(err) - nr += na - } - if nr != n { - panic("short read") - } - - b := make([]byte, n) - copy(b, a) - if !bytes.Equal(a, b) { - panic("test prep failed") - } - - m := &MultiReaderB{ - a: ioutil.NopCloser(bytes.NewBuffer(a)), - b: ioutil.NopCloser(bytes.NewBuffer(b)), - } - - // should not trigger the internal panic of MultiReadB - ncp, err := io.Copy(ioutil.Discard, m) - PanicOn(err) - if ncp != int64(n) { - panic("short copy") - } - - for victim := 0; victim < n; victim += 7 { - - copy(b, a) - if victim%2 == 0 { - // corrupt b - b[victim] = (b[victim] + 1) % 255 - } else { - // corrupt a - a[victim] = (a[victim] + 1) % 255 - } - m = &MultiReaderB{ - a: ioutil.NopCloser(bytes.NewBuffer(a)), - b: ioutil.NopCloser(bytes.NewBuffer(b)), - } - helperShouldPanicOnCopy(m) - } - } -} - -func helperShouldPanicOnCopy(m *MultiReaderB) { - // differences in bytes read should be noticed - defer func() { - r := recover() - if r == nil { - panic("expected panic on byte difference but didn't see it") - } - }() - _, _ = io.Copy(ioutil.Discard, m) -} diff --git a/bolt.go b/bolt.go deleted file mode 100644 index 12435abef..000000000 --- a/bolt.go +++ /dev/null @@ -1,1612 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "math" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/molecula/featurebase/v2/hash" - "github.com/molecula/featurebase/v2/roaring" - "github.com/molecula/featurebase/v2/storage" - . "github.com/molecula/featurebase/v2/vprint" - - // On Bolt only, we still use the long txkey, because - // this allows Max() to work readily. - // - "github.com/molecula/featurebase/v2/short_txkey" - txkey "github.com/molecula/featurebase/v2/txkey" - "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" -) - -const isDebugRun = false - -// boltRegistrar facilitates shutdown -// of all the bolt databases started under -// tests. Its needed because most tests don't cleanup -// the *Index(es) they create. But we still -// want to shutdown boltDB goroutines -// after tests run. -// -// It also allows opening the same path twice to -// result in sharing the same open database handle, and -// thus the same transactional guarantees. -// -type boltRegistrar struct { - mu sync.Mutex - mp map[*BoltWrapper]bool - - path2db map[string]*BoltWrapper -} - -func (r *boltRegistrar) Size() int { - r.mu.Lock() - defer r.mu.Unlock() - nmp := len(r.mp) - npa := len(r.path2db) - if nmp != npa { - panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa)) - } - return nmp -} - -var globalBoltReg *boltRegistrar = newBoltTestRegistrar() - -var globalNextTxSnBolt int64 - -func newBoltTestRegistrar() *boltRegistrar { - - return &boltRegistrar{ - mp: make(map[*BoltWrapper]bool), - path2db: make(map[string]*BoltWrapper), - } -} - -// register each bolt created under tests, so we -// can clean them up. This is called by openBoltWrapper() while -// holding the r.mu.Lock, since it needs to atomically -// check the registry and make a new instance only -// if one does not exist for its path, and otherwise -// return the existing instance. -func (r *boltRegistrar) unprotectedRegister(w *BoltWrapper) { - r.mp[w] = true - r.path2db[w.path] = w -} - -// unregister removes w from r -func (r *boltRegistrar) unregister(w *BoltWrapper) { - r.mu.Lock() - delete(r.mp, w) - delete(r.path2db, w.path) - r.mu.Unlock() -} - -func DumpAllBolt() { - short := true - globalBoltReg.mu.Lock() - defer globalBoltReg.mu.Unlock() - for w := range globalBoltReg.mp { - AlwaysPrintf("this bolt path='%v' has: \n%v\n", w.path, w.StringifiedBoltKeys(nil, short)) - } -} - -// openBoltDB opens the database in the bpath directoy -// without deleting any prior content. Any BoltDB -// database directory will have the "-bolt" suffix. -// -// openBoltDB will check the registry and make a new instance only -// if one does not exist for its bpath. Otherwise it returns -// the existing instance. This insures only one boltDB -// per bpath in this pilosa node. -func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error) { - r.mu.Lock() - defer r.mu.Unlock() - w, ok := r.path2db[path] - if ok { - // creates the effect of having only one bolt open per pilosa node. - return w, nil - } - // otherwise, make a new bolt and store it in globalBoltReg - - dir := filepath.Dir(path) - if !DirExists(path) { - PanicOn(os.MkdirAll(dir, 0755)) - } - fsyncEnabled := true - if cfg != nil { - fsyncEnabled = cfg.FsyncEnabled - } - - db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !fsyncEnabled}) - if err != nil { - return nil, errors.Wrapf(err, fmt.Sprintf("open bolt path '%v'", path)) - } - - // docs on fsync from https://godoc.org/github.com/etcd-io/bbolt - // - // Setting the NoSync flag will cause the database to skip fsync() - // calls after each commit. This can be useful when bulk loading data - // into a database and you can restart the bulk load in the event of - // a system failure or database corruption. Do not set this flag for - // normal use. - // - // If the package global IgnoreNoSync constant is true, this value is - // ignored. See the comment on that constant for more details. - // - // THIS IS UNSAFE. PLEASE USE WITH CAUTION. - // NoSync bool - - // When true, skips syncing freelist to disk. This improves the database - // write performance under normal operation, but requires a full database - // re-sync during recovery. - // NoFreelistSync bool - - if cfg != nil && !cfg.FsyncEnabled { - db.NoSync = true - db.NoFreelistSync = true - } else { - // default to using fsync on bolt. - db.NoSync = false - db.NoFreelistSync = false - } - - err = db.Update(func(tx *bolt.Tx) (err error) { - _, err = tx.CreateBucketIfNotExists(bucketCT) - return - }) - if err != nil { - return nil, errors.Wrapf(err, fmt.Sprintf("create bolt bucket '%v' in path '%v'", string(bucketCT), path)) - } - - name := filepath.Base(path) - w = &BoltWrapper{ - name: name, - db: db, - reg: r, - path: path, - doAllocZero: doAllocZero, - openTx: make(map[*BoltTx]bool), - - DeleteEmptyContainer: true, - fsyncEnabled: cfg.FsyncEnabled, - } - r.unprotectedRegister(w) - - return w, nil -} - -func (w *BoltWrapper) Path() string { - return w.path -} - -func (w *BoltWrapper) HasData() (has bool, err error) { - - tx, err := w.NewTx(!writable, "", Txo{}) - if err != nil { - return false, errors.Wrap(err, "HasData NewTx") - } - defer tx.Rollback() - - bi := NewBoltIterator(tx.(*BoltTx), nil) - defer bi.Close() - - for bi.Next() { - return true, nil - } - return false, nil -} - -func (w *BoltWrapper) CleanupTx(tx Tx) { - // inlined into Rollback and Commit, so this is a no-op, just here to satisfy the interface. -} -func (w *BoltWrapper) CloseDB() error { - w.muDb.Lock() - defer w.muDb.Unlock() - w.closed = true - return w.db.Close() -} -func (w *BoltWrapper) OpenDB() error { - w.muDb.Lock() - defer w.muDb.Unlock() - db, err := bolt.Open(w.path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !w.fsyncEnabled}) - if err != nil { - return err - } - w.db = db - w.closed = false - return nil -} - -func (tx *BoltTx) IsDone() (done bool) { - return atomic.LoadInt64(&tx.unlocked) == 1 -} - -func (w *BoltWrapper) OpenListString() (r string) { - - list := w.listopen() - if len(list) == 0 { - return "" - } - for i, ltx := range list { - if ltx.o.Write { - r += fmt.Sprintf("[%v]write: _sn_ %v %v, \n", i, ltx.sn, ltx.o) - } else { - r += fmt.Sprintf("[%v]read : _sn_ %v %v, \n", i, ltx.sn, ltx.o) - } - } - return -} - -func (w *BoltWrapper) listopen() (slc []*BoltTx) { - w.muDb.Lock() - for v := range w.openTx { - slc = append(slc, v) - } - w.muDb.Unlock() - return -} - -func (w *BoltWrapper) OpenSnList() (slc []int64) { - w.muDb.Lock() - for v := range w.openTx { - slc = append(slc, v.sn) - } - w.muDb.Unlock() - return -} - -// DeleteIndex deletes all the containers associated with -// the named index from the bolt database. -func (w *BoltWrapper) DeleteIndex(indexName string) error { - - // We use the apostrophie rune `'` to locate the end of the - // index name in the key prefix, so we cannot allow indexNames - // themselves to contain apostrophies. - if strings.Contains(indexName, "/") { - return fmt.Errorf("error: bad indexName `%v` in BoltWrapper.DeleteIndex() call: indexName cannot contain '/'", indexName) - } - prefix := txkey.IndexOnlyPrefix(indexName) - return w.DeletePrefix(prefix) -} - -// statically confirm that BoltTx satisfies the Tx interface. -var _ Tx = (*BoltTx)(nil) - -// BoltWrapper provides the NewTx() method. -type BoltWrapper struct { - db *bolt.DB - - muDb sync.Mutex - - path string - name string - - // track our registrar for Close / goro leak reporting purposes. - reg *boltRegistrar - - // make BoltWrapper.Close() idempotent, avoiding panic on double Close() - closed bool - - // doAllocZero sets the corresponding flag on all new BoltTx. - // When doAllocZero is true, we zero out any data from bolt - // after transcation commit and rollback. This simulates - // what would happen if we were to use the mmap-ed data - // from bolt directly. Currently we copy by default for - // safety because otherwise TestAPI_ImportColumnAttrs sees - // corrupted data. - doAllocZero bool - - DeleteEmptyContainer bool - fsyncEnabled bool // for tracking whether our initial config wanted fsync on - - openTx map[*BoltTx]bool -} - -func (w *BoltWrapper) SetHolder(h *Holder) { - // don't need it at the moment - //w.h = h -} - -// NewTxWRITE lets us see in the callstack dumps where the WRITE tx are. -// Can't have more than one active write per database, so the -// 2nd one will block until the first finishes. -func (w *BoltWrapper) NewTxWRITE() (*bolt.Tx, error) { - boltTxn, err := w.db.Begin(true) - if err != nil { - if w.db == nil || w.IsClosed() { - return nil, fmt.Errorf("cannot call NewTxWRITE() on closed Bolt database: '%v'", err) - } - return nil, err - } - return boltTxn, nil -} - -// NewTxREAD lets us see in the callstack dumps where the READ tx are. -func (w *BoltWrapper) NewTxREAD() (*bolt.Tx, error) { - boltTxn, err := w.db.Begin(false) - if err != nil { - if w.db == nil || w.IsClosed() { - return nil, fmt.Errorf("cannot call NewTxREAD() on closed Bolt database: '%v'", err) - } - return nil, err - } - return boltTxn, nil -} - -// NewTx produces Bolt based transactions. If -// the transaction will modify data, then the write flag must be true. -// Read-only queries should set write to false, to allow more concurrency. -// Methods on a BoltTx are thread-safe, and can be called from -// different goroutines. -// -// initialIndexName is optional. It is set by the TxFactory from the Txo -// options provided at the Tx creation point. It allows us to recognize -// and isolate cross-index queries more quickly. It can always be empty "" -// but when set is highly useful for debugging. It has no impact -// on transaction behavior. -// -func (w *BoltWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { - - sn := atomic.AddInt64(&globalNextTxSnBolt, 1) - - ////vv("bolt new tx _sn_ %v; openTx='%v', stack \n%v", sn, w.OpenListString(), stack()) - ////vv("bolt new (write=%v, shard=%v) tx _sn_ %v; openTx='%v'", write, o.Shard, sn, w.OpenListString()) - - var boltTxn *bolt.Tx - if write { - // see the WRITE tx on the callstack. - boltTxn, err = w.NewTxWRITE() - if err != nil { - return nil, err - } - } else { - // see the READ tx on the callstack. - boltTxn, err = w.NewTxREAD() - if err != nil { - return nil, err - } - } - - ltx := &BoltTx{ - sn: sn, - write: write, - tx: boltTxn, - Db: w, - frag: o.Fragment, - doAllocZero: w.doAllocZero, - initialIndexName: initialIndexName, - DeleteEmptyContainer: w.DeleteEmptyContainer, - o: o, - gid: curGID(), - } - tx = ltx - - if isDebugRun { - w.muDb.Lock() - w.openTx[ltx] = true - w.muDb.Unlock() - } - return -} - -// Close shuts down the Bolt database. -func (w *BoltWrapper) Close() (err error) { - w.muDb.Lock() - defer w.muDb.Unlock() - if !w.closed { - if isDebugRun { - // complain if there are still Tx in flight, b/c otherwise we will see - // the somewhat mysterious 'panic: should not be in ReadSlot.free() with slot still owned by gid=107043; refCount=1' - if len(w.openTx) > 0 { - AlwaysPrintf("error: cannot close BoltWrapper with Tx still in flight.") - return - } - } - w.reg.unregister(w) - w.closed = true - return w.db.Close() - } - return nil -} - -func (w *BoltWrapper) IsClosed() (closed bool) { - w.muDb.Lock() - closed = w.closed - w.muDb.Unlock() - return -} - -// BoltTx wraps a bolt.Tx and provides the Tx interface -// method implementations. -// The methods on BoltTx are thread-safe, and can be called -// from different goroutines. -type BoltTx struct { - - // mu serializes bolt operations on this single txn instance. - mu sync.Mutex - sn int64 // serial number - - write bool - Db *BoltWrapper - tx *bolt.Tx - frag *fragment - - opcount int - - //initloc string // stack trace of where we were initially created. - - doAllocZero bool - - initialIndexName string - - DeleteEmptyContainer bool - - unlocked int64 - - o Txo - - // NewTx, write operations, Commit and/or Rollback must all take place on - // the same gid and it must the runtime.LockOSThreaded first. Verify - // that we are using the right goroutine in a debug build using the - // gid, stored here, used for NewTx(). - gid uint64 -} - -// sanity check that database is open. -func (tx *BoltTx) sanity() { - if tx.Db.IsClosed() { - panic("cannot operate on closed Bolt") - } -} - -func (tx *BoltTx) Group() *TxGroup { - return tx.o.Group -} - -func (tx *BoltTx) Type() string { - return BoltTxn -} - -func (tx *BoltTx) UseRowCache() bool { - return storage.EnableRowCache() -} - -// Pointer gives us a memory address for the underlying transaction for debugging. -// It is public because we use it in roaring to report invalid container memory access -// outside of a transaction. -func (tx *BoltTx) Pointer() string { - return fmt.Sprintf("%p", tx) -} - -// Rollback rolls back the transaction. -func (tx *BoltTx) Rollback() { - notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) - if !notDone { - return - } - ////vv("bolt rollback tx _sn_ %v; stack \n%v", tx.sn) // , stack()) - if isDebugRun { - tx.sanity() - tx.Db.muDb.Lock() - delete(tx.Db.openTx, tx) - tx.Db.muDb.Unlock() - } - - tx.mu.Lock() - defer tx.mu.Unlock() - - //tx.debugOnlyGidcheck() - _ = tx.tx.Rollback() // must hold tx.mu mutex lock - - tx.o.dbs.Cleanup(tx) -} - -// Commit commits the transaction to permanent storage. -// Commits can handle up to 100k updates to fragments -// at once, but not more. This is a BoltDB imposed limit. -func (tx *BoltTx) Commit() error { - notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) - if !notDone { - ////vv("Commit already done") - return nil - } - ////vv("bolt commit tx _sn_ %v; path = '%v'; stack \n%v", tx.sn, tx.Db.path, stack()) - //DumpAllBolt() - - if isDebugRun { - tx.sanity() - tx.Db.muDb.Lock() - delete(tx.Db.openTx, tx) - tx.Db.muDb.Unlock() - } - tx.mu.Lock() - defer tx.mu.Unlock() - - err := tx.tx.Commit() - PanicOn(err) - - tx.o.dbs.Cleanup(tx) - return err -} - -// Readonly returns true iff the BoltTx is read-only. -func (tx *BoltTx) Readonly() bool { - return !tx.write -} - -// RoaringBitmap returns the roaring.Bitmap for all bits in the fragment. -func (tx *BoltTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - - return tx.OffsetRange(index, field, view, shard, 0, 0, LeftShifted16MaxContainerKey) -} - -// Container returns the requested roaring.Container, selected by fragment and ckey -func (tx *BoltTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) { - - // values returned from Get() are only valid while the transaction - // is open. If you need to use a value outside of the transaction then - // you must use copy() to copy it to another byte slice. - // BUT here we are already inside the Txn. - - bkey := txkey.Key(index, field, view, shard, ckey) - tx.mu.Lock() - - //tx.debugOnlyGidcheck() - - bkt := tx.tx.Bucket(bucketCT) - v := bkt.Get(bkey) - tx.mu.Unlock() - - if v == nil { - // not found - return nil, nil - } - n := len(v) - if n > 0 { - c = tx.toContainer(v[n-1], v[0:(n-1)]) - } - return -} - -var bucketCT = []byte("ct") - -// PutContainer stores rc under the specified fragment and container ckey. -func (tx *BoltTx) PutContainer(index, field, view string, shard uint64, ckey uint64, rc *roaring.Container) error { - - bkey := txkey.Key(index, field, view, shard, ckey) - var by []byte - - ct := roaring.ContainerType(rc) - - switch ct { - case roaring.ContainerArray: - by = fromArray16(roaring.AsArray(rc)) - case roaring.ContainerBitmap: - by = fromArray64(roaring.AsBitmap(rc)) - case roaring.ContainerRun: - by = fromInterval16(roaring.AsRuns(rc)) - case roaring.ContainerNil: - panic("wat? nil container is unexpected, no?!?") - default: - panic(fmt.Sprintf("unknown container type: %v", ct)) - } - tx.mu.Lock() - - bkt := tx.tx.Bucket(bucketCT) - err := bkt.Put(bkey, append(by, ct)) - ////vv("err on put bkey = '%v' was %v", string(bkey), err) - tx.mu.Unlock() - - return err -} - -// RemoveContainer deletes the container specified by the shard and container key ckey -func (tx *BoltTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error { - bkey := txkey.Key(index, field, view, shard, ckey) - tx.mu.Lock() - //tx.debugOnlyGidcheck() - - bkt := tx.tx.Bucket(bucketCT) - err := bkt.Delete(bkey) - - tx.mu.Unlock() - return err -} - -// Add sets all the a bits hot in the specified fragment. -func (tx *BoltTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - return tx.addOrRemove(index, field, view, shard, false, a...) -} - -// Remove clears all the specified a bits in the chosen fragment. -func (tx *BoltTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - return tx.addOrRemove(index, field, view, shard, true, a...) -} - -func (tx *BoltTx) addOrRemove(index, field, view string, shard uint64, remove bool, a ...uint64) (changeCount int, err error) { - if len(a) == 0 { - return 0, nil - } - - // have to sort, b/c input is not always sorted. - sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) - - var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. - var rc *roaring.Container - var hi uint64 - var lo uint16 - - for i, v := range a { - - hi, lo = highbits(v), lowbits(v) - if hi != lastHi { - // either first time through, or changed to a different container. - // do we need put the last updated container now? - if i > 0 { - // not first time through, write what we got. - if remove && (rc == nil || rc.N() == 0) { - err = tx.RemoveContainer(index, field, view, shard, lastHi) - PanicOn(err) - } else { - err = tx.PutContainer(index, field, view, shard, lastHi, rc) - PanicOn(err) - } - } - // get the next container - rc, err = tx.Container(index, field, view, shard, hi) - PanicOn(err) - } // else same container, keep adding bits to rct. - chng := false - // rc can be nil before, and nil after, in both Remove/Add below. - // The roaring container add() and remove() methods handle this. - if remove { - rc, chng = rc.Remove(lo) - } else { - rc, chng = rc.Add(lo) - } - if chng { - changeCount++ - } - lastHi = hi - } - // write the last updates. - if remove { - if rc == nil || rc.N() == 0 { - err = tx.RemoveContainer(index, field, view, shard, hi) - PanicOn(err) - } else { - err = tx.PutContainer(index, field, view, shard, hi, rc) - PanicOn(err) - } - } else { - if rc == nil || rc.N() == 0 { - panic("there should be no way to have an empty bitmap AFTER an Add() operation") - } - err = tx.PutContainer(index, field, view, shard, hi, rc) - PanicOn(err) - } - return -} - -// Contains returns exists true iff the bit chosen by key is -// hot (set to 1) in specified fragment. -func (tx *BoltTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { - - lo, hi := lowbits(key), highbits(key) - bkey := txkey.Key(index, field, view, shard, hi) - tx.mu.Lock() - - bkt := tx.tx.Bucket(bucketCT) - v := bkt.Get(bkey) - - tx.mu.Unlock() - if v == nil { - return false, nil - } - n := len(v) - if n > 0 { - c := tx.toContainer(v[n-1], v[0:(n-1)]) - exists = c.Contains(lo) - } - return exists, err -} - -// key is the container key for the first roaring Container -// roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will -// return the first container at or after key. found will be true if a -// container is found at key. -// -// BoltTx notes: We auto-stop at the end of this shard, not going beyond. -func (tx *BoltTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { - - // needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000" - needle := txkey.Key(index, field, view, shard, firstRoaringContainerKey) - - // prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@" - prefix := txkey.Prefix(index, field, view, shard) - - bi := NewBoltIterator(tx, prefix) - ok := bi.Seek(needle) - if !ok { - return bi, false, nil - } - - // have to compare b/c bolt might give us valid iterator - // that is past our needle if needle isn't present. - return bi, bytes.Equal(bi.lastKey, needle), nil -} - -func (tx *BoltTx) GetFieldSizeBytes(index, field string) (uint64, error) { - return 0, nil -} - -// BoltIterator is the iterator returned from a BoltTx.ContainerIterator() call. -// It implements the roaring.ContainerIterator interface. -type BoltIterator struct { - tx *BoltTx - cur *bolt.Cursor - - prefix []byte - seekto []byte - - // seen counts how many Next() calls we have seen. - // It is used to match roaring.ContainerIterator semantics. - // Also useful for testing. - seen int - - lastKey []byte - lastVal []byte // *roaring.Container - lastOK bool - lastConsumed bool -} - -// NewBoltIterator creates an iterator on tx that will -// only return boltKeys that start with prefix. -func NewBoltIterator(tx *BoltTx, prefix []byte) (bi *BoltIterator) { - - tx.mu.Lock() - - bkt := tx.tx.Bucket(bucketCT) - cur := bkt.Cursor() - tx.mu.Unlock() - - bi = &BoltIterator{ - tx: tx, - cur: cur, - prefix: prefix, - } - - return -} - -// Close tells the database and transaction that the user is done -// with the iterator. -func (bi *BoltIterator) Close() { - // no-op -} - -// Valid returns false if there are no more values in the iterator's range. -func (bi *BoltIterator) Valid() bool { - return bi.lastOK -} - -// Seek allows the iterator to start at needle instead of the global begining. -func (bi *BoltIterator) Seek(needle []byte) (ok bool) { - bi.tx.mu.Lock() - defer bi.tx.mu.Unlock() - - bi.seen++ // if ommited, red TestBolt_ContainerIterator_empty_iteration_loop() - - k, v := bi.cur.Seek(needle) - ////vv("seek to needle '%v' gives key '%v'", string(needle), txkey.ToString(k)) - - if len(k) == 0 { - // not found, no keys after needle. - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - if len(bi.prefix) > 0 { - ok = bytes.HasPrefix(k, bi.prefix) - if !ok { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - } - - bi.lastKey = k - bi.lastVal = v - bi.lastOK = true - bi.lastConsumed = false - - return true -} - -func (bi *BoltIterator) ValidForPrefix(prefix []byte) bool { - if !bi.lastOK { - return false - } - if len(bi.prefix) == 0 { - return true - } - return bytes.HasPrefix(bi.lastKey, bi.prefix) -} - -func (bi *BoltIterator) String() (r string) { - return fmt.Sprintf("BoltIterator{prefix: '%v', seekto: '%v', seen:%v, lastKey:'%v', lastOK:%v, lastConsumed:%v}", string(bi.prefix), string(bi.seekto), bi.seen, string(bi.lastKey), bi.lastOK, bi.lastConsumed) -} - -// Next advances the iterator. -func (bi *BoltIterator) Next() (ok bool) { - //vv("Next; bi.prefix='%v'", string(bi.prefix)) - if bi.lastOK && !bi.lastConsumed { - //vv("lastOk and not consumed, returning wo doing anything") - bi.seen++ - bi.lastConsumed = true - if len(bi.lastVal) == 0 { - panic("bi.lastVal should not have len 0 if lastOK true") - } - return true - } - - var k, v []byte - if bi.seen == 0 { - if len(bi.prefix) == 0 { - bi.tx.mu.Lock() - k, v = bi.cur.First() - bi.tx.mu.Unlock() - } else { - found := bi.Seek(bi.prefix) - // increments bi.seen for us. - if !found { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - // ready to go - return true - } - } - - bi.seen++ -skipEmpty: - if bi.seen > 1 { - bi.tx.mu.Lock() - k, v = bi.cur.Next() - bi.tx.mu.Unlock() - } - - if len(k) == 0 { - // no more - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - if len(bi.prefix) > 0 { - ok = bytes.HasPrefix(k, bi.prefix) - if !ok { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - } - bi.lastKey = k - bi.lastVal = v - if len(v) == 0 { - // actually under !tx.DeleteEmptyContainer, we can have empty containers! - goto skipEmpty - } - bi.lastOK = true - bi.lastConsumed = true - - return true -} - -// Value retrieves what is pointed at currently by the iterator. -func (bi *BoltIterator) Value() (containerKey uint64, c *roaring.Container) { - if !bi.lastOK { - panic("bi.cur not valid") - } - containerKey = txkey.KeyExtractContainerKey(bi.lastKey) - - v := bi.lastVal - n := len(v) - if n > 0 { - c = bi.tx.toContainer(v[n-1], v[0:(n-1)]) - } else { - panic("v should not be empty!") - } - return -} - -// boltFinder implements roaring.IteratorFinder. -// It is used by BoltTx.ForEach() -type boltFinder struct { - tx *BoltTx - index string - field string - view string - shard uint64 - needClose []Closer -} - -// FindIterator lets boltFinder implement the roaring.FindIterator interface. -func (bf *boltFinder) FindIterator(seek uint64) (roaring.ContainerIterator, bool) { - a, found, err := bf.tx.ContainerIterator(bf.index, bf.field, bf.view, bf.shard, seek) - PanicOn(err) - bf.needClose = append(bf.needClose, a) - return a, found -} - -// Close closes all bf.needClose listed Closers. -func (bf *boltFinder) Close() { - for _, i := range bf.needClose { - i.Close() - } -} - -// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE -// the transaction Commits or Rollsback. -func (tx *BoltTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { - - bf := &boltFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} - itr := roaring.NewIterator(bf) - return itr -} - -// ForEach applies fn to each bitmap in the fragment. -func (tx *BoltTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { - - itr := tx.NewTxIterator(index, field, view, shard) - defer itr.Close() - - // Seek can create many container iterators, thus bf.Close() needClose list. - itr.Seek(0) - // v is the bit we are operating on. - for v, eof := itr.Next(); !eof; v, eof = itr.Next() { - if err := fn(v); err != nil { - return err - } - } - return nil -} - -// ForEachRange applies fn on the selected range of bits on the chosen fragment. -func (tx *BoltTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { - - itr := tx.NewTxIterator(index, field, view, shard) - defer itr.Close() - - itr.Seek(start) - - // v is the bit we are operating on. - for v, eof := itr.Next(); !eof && v < end; v, eof = itr.Next() { - if err := fn(v); err != nil { - return err - } - } - return nil -} - -// Count operates on the full bitmap level, so it sums over all the containers -// in the bitmap. -func (tx *BoltTx) Count(index, field, view string, shard uint64) (uint64, error) { - - a, found, err := tx.ContainerIterator(index, field, view, shard, 0) - PanicOn(err) - defer a.Close() - if !found { - return 0, nil - } - result := int32(0) - for a.Next() { - ckey, cont := a.Value() - _ = ckey - result += cont.N() - } - return uint64(result), nil -} - -// Max is the maximum bit-value in your bitmap. -// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does. -func (tx *BoltTx) Max(index, field, view string, shard uint64) (uint64, error) { - - prefix := txkey.Prefix(index, field, view, shard) - seekto := txkey.Prefix(index, field, view, shard+1) - - bkt := tx.tx.Bucket(bucketCT) - cur := bkt.Cursor() - - var k, v []byte - k, _ = cur.Seek(seekto) - if k == nil { - // we have nothing >= seekto, but we might have stuff before it, and we'll wrap backwards. - k, v = cur.Prev() - if k == nil { - // empty database - return 0, nil - } - } else { - // we found something >= seekto, so backup by 1. - k, v = cur.Prev() - if k == nil { - // nothing before seekto - return 0, nil - } - } - - // have something, are we in [prefix, seekto) ? - cmp := bytes.Compare(k, prefix) - if cmp >= 0 { - // good, got max in k, v - } else { - return 0, nil // nothing in [prefix, seekto). - } - - n := len(v) - if n == 0 { - return 0, nil - } - - hb := txkey.KeyExtractContainerKey(k) - rc := tx.toContainer(v[n-1], v[0:(n-1)]) - - lb := rc.Max() - return hb<<16 | uint64(lb), nil -} - -// Min returns the smallest bit set in the fragment. If no bit is hot, -// the second return argument is false. -func (tx *BoltTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { - - // Seek can create many container iterators, thus the bf.Close() needClose list. - bf := &boltFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} - defer bf.Close() - itr := roaring.NewIterator(bf) - - itr.Seek(0) - - // v is the bit we are operating on. - v, eof := itr.Next() - if eof { - return 0, false, nil - } - return v, true, nil -} - -// UnionInPlace unions all the others Bitmaps into a new Bitmap, and then writes it to the -// specified fragment. -func (tx *BoltTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - PanicOn(err) - - rbm.UnionInPlace(others...) - // iterate over the containers that changed within rbm, and write them back to disk. - - it, found := rbm.Containers.Iterator(0) - _ = found // don't care about the value of found, because first containerKey might be > 0 - - for it.Next() { - containerKey, rc := it.Value() - - // TODO: only write the changed ones back, as optimization? - // Compare to ImportRoaringBits. - err := tx.PutContainer(index, field, view, shard, containerKey, rc) - PanicOn(err) - } - return nil -} - -// CountRange returns the count of hot bits in the start, end range on the fragment. -// roaring.countRange counts the number of bits set between [start, end). -func (tx *BoltTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { - - if start >= end { - return 0, nil - } - - skey := highbits(start) - ekey := highbits(end) - - citer, found, err := tx.ContainerIterator(index, field, view, shard, skey) - _ = found - PanicOn(err) - - defer citer.Close() - - // If range is entirely in one container then just count that range. - if skey == ekey { - citer.Next() - _, c := citer.Value() - return uint64(c.CountRange(int32(lowbits(start)), int32(lowbits(end)))), nil - } - - for citer.Next() { - k, c := citer.Value() - if k < skey { - citer.Close() - panic(fmt.Sprintf("should be impossible for k(%v) to be less than skey(%v). tx p=%p", k, skey, tx)) - } - - // k > ekey handles the case when start > end and where start and end - // are in different containers. Same container case is already handled above. - if k > ekey { - break - } - if k == skey { - n += uint64(c.CountRange(int32(lowbits(start)), roaring.MaxContainerVal+1)) - continue - } - if k < ekey { - n += uint64(c.N()) - continue - } - if k == ekey { - n += uint64(c.CountRange(0, int32(lowbits(end)))) - break - } - } - - return n, nil -} - -// OffsetRange creates a new roaring.Bitmap to return in other. For all the -// hot bits in [start, endx) of the chosen fragment, it stores -// them into other but with offset added to their bit position. -// The primary client is doing this, using ShardWidth, already; see -// fragment.rowFromStorage() in fragment.go. For example: -// -// data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard, -// -// f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) -// ^ offset ^ start ^ endx -// -// The start and endx arguments are container keys that have been shifted left by 16 bits; -// their highbits() will be taken to determine the actual container keys. This -// is done to conform to the roaring.OffsetRange() argument convention. -// -func (tx *BoltTx) OffsetRange(index, field, view string, shard, offset, start, endx uint64) (other *roaring.Bitmap, err error) { - ////vv("top of BoltTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v)", index, field, view, int(shard), int(offset), int(start), int(endx)) - //defer func() { - ////vv("returning from BoltTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) other returning is: '%#v' stack=\n%v", index, field, view, int(shard), int(offset), int(start), int(endx), asInts(other.Slice()), stack()) - //}() - - // roaring does these three checks in its OffsetRange - if lowbits(offset) != 0 { - panic("offset must not contain low bits") - } - if lowbits(start) != 0 { - panic("range start must not contain low bits") - } - if lowbits(endx) != 0 { - panic("range end must not contain low bits") - } - - other = roaring.NewSliceBitmap() - off := highbits(offset) - hi0, hi1 := highbits(start), highbits(endx) - - needle := txkey.Key(index, field, view, shard, hi0) - prefix := txkey.Prefix(index, field, view, shard) - - it := NewBoltIterator(tx, prefix) - defer it.Close() - it.Seek(needle) - for ; it.ValidForPrefix(prefix); it.Next() { - bkey := it.lastKey - k := txkey.KeyExtractContainerKey(bkey) - - // >= hi1 is correct b/c endx cannot have any lowbits set. - if uint64(k) >= hi1 { - break - } - destCkey := off + (k - hi0) - - v := it.lastVal - n := len(v) - if n == 0 { - continue - } - c := tx.toContainer(v[n-1], v[0:(n-1)]) - other.Containers.Put(destCkey, c.Freeze()) - } - return other, nil -} - -// IncrementOpN increments the tx opcount by changedN -func (tx *BoltTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - tx.opcount += changedN -} - -// ImportRoaringBits handles deletes by setting clear=true. -// rowSet[rowID] returns the number of bit changed on that rowID. -func (tx *BoltTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { - - n := itr.Len() - if n == 0 { - return - } - rowSet = make(map[uint64]int) - - var currRow uint64 - - var oldC *roaring.Container - for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { - if rowSize != 0 { - currRow = itrKey / rowSize - } - nsynth := int(synthC.N()) - if nsynth == 0 { - continue - } - // INVAR: nsynth > 0 - - oldC, err = tx.Container(index, field, view, shard, itrKey) - PanicOn(err) - if err != nil { - return - } - - if oldC == nil || oldC.N() == 0 { - // no container at the itrKey in bolt (or all zero container). - if clear { - // changed of 0 and empty rowSet is perfect, no need to change the defaults. - continue - } else { - - changed += nsynth - rowSet[currRow] += nsynth - - err = tx.PutContainer(index, field, view, shard, itrKey, synthC) - if err != nil { - return - } - continue - } - } - - if clear { - existN := oldC.N() // number of bits set in the old container - newC := oldC.Difference(synthC) - - // update rowSet and changes - if newC.N() == existN { - // INVAR: do changed need adjusting? nope. same bit count, - // so no change could have happened. - continue - } else { - changes := int(existN - newC.N()) - changed += changes - rowSet[currRow] -= changes - - if tx.DeleteEmptyContainer && newC.N() == 0 { - err = tx.RemoveContainer(index, field, view, shard, itrKey) - if err != nil { - return - } - continue - } - err = tx.PutContainer(index, field, view, shard, itrKey, newC) - if err != nil { - return - } - continue - } - } else { - // setting bits - - existN := oldC.N() - if existN == roaring.MaxContainerVal+1 { - // completely full container already, set will do nothing. so changed of 0 default is perfect. - continue - } - if existN == 0 { - // can nsynth be zero? No, because of the continue/invariant above where nsynth > 0 - changed += nsynth - rowSet[currRow] += nsynth - err = tx.PutContainer(index, field, view, shard, itrKey, synthC) - if err != nil { - return - } - continue - } - - newC := roaring.Union(oldC, synthC) // UnionInPlace was giving us crashes on overly large containers. - - if roaring.ContainerType(newC) == roaring.ContainerBitmap { - newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it. - } - if newC.N() != existN { - changes := int(newC.N() - existN) - changed += changes - rowSet[currRow] += changes - - err = tx.PutContainer(index, field, view, shard, itrKey, newC) - if err != nil { - PanicOn(err) - return - } - continue - } - } - } - return -} - -func (tx *BoltTx) toContainer(typ byte, v []byte) (r *roaring.Container) { - - //tx.debugOnlyGidcheck() - - if len(v) == 0 { - return nil - } - - var w []byte - useRowCache := tx.UseRowCache() - if tx.doAllocZero || useRowCache { - // Do electric fence-inspired bad-memory read detection. - // - // The v []byte lives in BoltDB's memory-mapped vlog-file, - // and Bolt will recycle it after tx ends with rollback or commit. - // - // Problem is, at least some operations were not respecting transaction boundaries. - // This technique helped us find them. The rowCache was an example. - // - // See the global const DetectMemAccessPastTx - // at the top of txfactory.go to activate/deactivate this. - // - // Seebs suggested this nice variation: we could use individual mmaps for these - // copies, which would be unusable in production, but workable for testing, and then unmap them, - // which would get us probable segfaults on future accesses to them. - // - // The go runtime also has an -efence flag which may be similarly useful if really pressed. - // - w = make([]byte, len(v)) - copy(w, v) - } else { - w = v - } - return ToContainer(typ, w) -} - -func ToContainer(typ byte, w []byte) (c *roaring.Container) { - switch typ { - case roaring.ContainerArray: - c = roaring.NewContainerArray(toArray16(w)) - case roaring.ContainerBitmap: - c = roaring.NewContainerBitmap(-1, toArray64(w)) - case roaring.ContainerRun: - c = roaring.NewContainerRun(toInterval16(w)) - default: - panic(fmt.Sprintf("unknown container: %v", typ)) - } - c.SetMapped(true) - return c -} - -// StringifiedBoltKeys returns a string with all the container -// keys available in bolt. -func (w *BoltWrapper) StringifiedBoltKeys(optionalUseThisTx Tx, short bool) (r string) { - if optionalUseThisTx == nil { - tx, _ := w.NewTx(!writable, "", Txo{}) - defer tx.Rollback() - r = stringifiedBoltKeysTx(tx.(*BoltTx), short) - return - } - - btx, ok := optionalUseThisTx.(*BoltTx) - if !ok { - return fmt.Sprintf("", optionalUseThisTx) - } - r = stringifiedBoltKeysTx(btx, short) - return -} - -// countBitsSet returns the number of bits set (or "hot") in -// the roaring container value found by the txkey.Key() -// formatted bkey. -func (tx *BoltTx) countBitsSet(bkey []byte) (n int) { - - //tx.debugOnlyGidcheck() - - bkt := tx.tx.Bucket(bucketCT) - v := bkt.Get(bkey) - - if v == nil { - // some queries bkey may not be present! don't panic. - return 0 - } - - n = len(v) - if n > 0 { - rc := tx.toContainer(v[n-1], v[0:(n-1)]) - n = int(rc.N()) - } - return -} - -func (tx *BoltTx) Dump(short bool, shard uint64) { - fmt.Printf("%v\n", stringifiedBoltKeysTx(tx, short)) -} - -func (tx *BoltTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []short_txkey.FieldView, err error) { - bkt := tx.tx.Bucket(bucketCT) - err = bkt.ForEach(func(bkey, v []byte) error { - fv := txkey.FieldViewFromFullKey(bkey) - var shortFV short_txkey.FieldView - shortFV.Field = fv.Field - shortFV.View = fv.View - fvs = append(fvs, shortFV) - return nil - }) - return -} - -// stringifiedBoltKeysTx reports all the bolt keys and a -// corresponding blake3 hash viewable by txn within the entire -// bolt database. -// It also reports how many bits are hot in the roaring container -// (how many bits are set, or 1 rather than 0). -// -// By convention, we must return the empty string if there -// are no keys present. The tests use this to confirm -// an empty database. -func stringifiedBoltKeysTx(tx *BoltTx, short bool) (r string) { - - r = "allkeys:[\n" - it := NewBoltIterator(tx, nil) - defer it.Close() - any := false - for it.Next() { - any = true - - bkey := it.lastKey - key := txkey.ToString(bkey) - ckey := txkey.KeyExtractContainerKey(bkey) - h := "" - srbm := "" - v := it.lastVal - n := len(v) - if n == 0 { - panic("should not have empty v here") - } - h = hash.Blake3sum16(v[0:(n - 1)]) - ct := tx.toContainer(v[n-1], v[0:(n-1)]) - cts := roaring.NewSliceContainers() - cts.Put(ckey, ct) - rbm := &roaring.Bitmap{Containers: cts} - srbm = BitmapAsString(rbm) - - r += fmt.Sprintf("%v -> %v (%v hot)\n", key, h, tx.countBitsSet(bkey)) - if !short { - r += " ......." + srbm + "\n" - } - } - r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) - - if !any { - return "" - } - return "bolt-" + r -} - -func (w *BoltWrapper) DeleteDBPath(dbs *DBShard) (err error) { - path := dbs.pathForType(boltTxn) - err = os.RemoveAll(path) - if err != nil { - return errors.Wrap(err, "DeleteDBPath") - } - return -} - -func (w *BoltWrapper) DeleteField(index, field, fieldPath string) (err error) { - - // TODO(jea) cleanup: I think this fieldPath delete just goes away now. - // remove this commented stuff once we are sure. - // - // under blue-green roaring_bolt, the directory will not be found, b/c roaring will have - // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: - // "If the path does not exist, RemoveAll returns nil (no error)" - - err = os.RemoveAll(fieldPath) - if err != nil { - return errors.Wrap(err, "removing directory") - } - prefix := txkey.FieldPrefix(index, field) - return w.DeletePrefix(prefix) -} - -func (w *BoltWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { - prefix := txkey.Prefix(index, field, view, shard) - return w.DeletePrefix(prefix) -} - -func (w *BoltWrapper) DeletePrefix(prefix []byte) error { - - tx, _ := w.NewTx(writable, w.name, Txo{}) - - // NewTx will grab these, so don't lock until after it. - w.muDb.Lock() - - bi := NewBoltIterator(tx.(*BoltTx), prefix) - - for bi.Next() { - //vv("deleting next in cur") - err := bi.cur.Delete() - if err != nil { - w.muDb.Unlock() - panic(err) - } - } - bi.Close() - - // Commit will grab the w.muDb lock, so we must release it first. - w.muDb.Unlock() - - err := tx.Commit() - PanicOn(err) - - return nil -} - -func (tx *BoltTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") - } - var buf bytes.Buffer - sz, err = rbm.WriteTo(&buf) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") - } - return ioutil.NopCloser(&buf), sz, err -} - -func (tx *BoltTx) Options() Txo { - return tx.o -} - -// Sn retreives the serial number of the Tx. -func (tx *BoltTx) Sn() int64 { - return tx.sn -} - -func (c *BoltTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { - return GenericApplyFilter(c, index, field, view, shard, ckey, filter) -} diff --git a/bolt_test.go b/bolt_test.go deleted file mode 100644 index 1dc03b7d0..000000000 --- a/bolt_test.go +++ /dev/null @@ -1,1270 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "fmt" - "math" - "os" - "testing" - - "github.com/molecula/featurebase/v2/roaring" - "github.com/molecula/featurebase/v2/storage" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck -) - -// helpers, each runs their own new txn, and commits if a change/delete -// was made. The txn is rolled back if it is just viewing the data. - -func BoltMustHaveBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, bitvalue uint64) { - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue '%v' was NOT SET!!!", bitvalue)) - } - - tx.Rollback() -} - -func BoltMustNotHaveBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, bitvalue uint64) { - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if exists { - panic(fmt.Sprintf("ARG bitvalue '%v' WAS SET but should not have been.!!!", bitvalue)) - } - tx.Rollback() -} - -func BoltMustSetBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - // add a bit - changed, err := tx.Add(index, field, view, shard, putme) - if changed != 1 { - panic("should have 1 bit changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic("ARG putme was NOT SET!!!") - } - PanicOn(tx.Commit()) -} - -func BoltMustDeleteBitvalueContainer(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - hi := highbits(putme) - PanicOn(tx.RemoveContainer(index, field, view, shard, hi)) - PanicOn(tx.Commit()) -} - -func BoltMustDeleteBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - _, err := tx.Remove(index, field, view, shard, putme) - PanicOn(err) - PanicOn(tx.Commit()) -} - -func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) { - var err error - fn := path - PanicOn(os.RemoveAll(fn)) - ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, &storage.Config{FsyncEnabled: false}) - PanicOn(err) - w = ww.(*BoltWrapper) - - // verify it is empty - allkeys := w.StringifiedBoltKeys(nil, false) - if allkeys != "" { - panic(fmt.Sprintf("freshly created database was not empty! had keys:'%v'", allkeys)) - } - - return w, func() { - w.Close() - PanicOn(os.RemoveAll(fn)) - } -} - -// end of helper utilities -////////////////////////// - -////////////////////////// -// begin Tx method tests - -func TestBolt_DeleteFragment(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_DeleteFragment") - defer clean() - defer dbwrap.Close() - index, field, shard := "i", "f", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} - views := []string{"v1", "v2"} - for _, view := range views { - for _, v := range bits { - changed, err := tx.Add(index, field, view, shard, v) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - } - } - - for _, view := range views { - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - } - } - err := tx.Commit() - PanicOn(err) - - // end of setup - - victim := "v1" - survivor := "v2" - err = dbwrap.DeleteFragment(index, field, victim, shard, nil) - PanicOn(err) - - tx, _ = dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - for _, view := range views { - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if view == survivor { - if !exists { - panic(fmt.Sprintf("ARG survivor died : bit %v", v)) - } - } else if view == victim { // victim, should have been deleted - if exists { - panic(fmt.Sprintf("ARG victim lived : bit %v", v)) - } - } - } - } -} - -func TestBolt_Max_on_many_containers(t *testing.T) { - path := "TestBolt_Max_on_many_containers" - dbwrap, clean := mustOpenEmptyBoltWrapper(path) - - defer clean() - defer dbwrap.Close() - index, field, view := "i", "f", "v" - - // 099 - // 101 - // 199 - // 300 - // 399 - // - // find max in [300,400) and get 399 - // find max in [000,100) and get 099 - // find max in [100,200) and get 199 - // find max in [400,500) and get nothing back - // find max in [200,300) and get nothing back - - shards := []int{99, 101, 199, 300, 399} - - for _, sh := range shards { - shard := uint64(sh) - for _, pm := range shards { - putme := uint64(pm) - if putme > shard { - continue - } - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - } - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - for _, shard := range shards { - max, err := tx.Max(index, field, view, uint64(shard)) - PanicOn(err) - if max != uint64(shard) { - panic(fmt.Sprintf("expected max (%v) to be == shard = %v", max, shard)) - } - } - - // check for not found - max, err := tx.Max(index, field, view, uint64(200)) - PanicOn(err) - if max != 0 { - panic("expected not found to give 0 max back with nil err") - } - max, err = tx.Max(index, field, view, uint64(400)) - PanicOn(err) - if max != 0 { - panic("expected not found to give 0 max back with nil err") - } - -} - -// and the rest - -func TestBolt_SetBitmap(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_SetBitmap") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - bitvalue := uint64(0) - changed, err := tx.Add(index, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - - err = tx.Commit() - PanicOn(err) - - // - // commited, so should be visible outside the txn - // - - tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) - exists, err = tx2.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!! on tx2") - } - - n, err := tx2.Count(index, field, view, shard) - PanicOn(err) - if n != 1 { - panic(fmt.Sprintf("should have Count 1; instead n = %v", n)) - } - tx2.Rollback() -} - -func TestBolt_OffsetRange(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_OffsetRange") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - bitvalue := uint64(1 << 20) - changed, err := tx.Add(index, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - bitvalue2 := uint64(1<<20 + 1) - changed, err = tx.Add(index, field, view, shard, bitvalue2) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - exists, err = tx.Contains(index, field, view, shard, bitvalue2) - PanicOn(err) - if !exists { - panic("ARG bitvalue2 was NOT SET!!!") - } - - err = tx.Commit() - PanicOn(err) - - offset := uint64(0 << 20) - start := uint64(0 << 16) - endx := bitvalue + 1<<16 - - tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) - rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx) - PanicOn(err) - tx2.Rollback() - - // should see our 1M value - s2 := BitmapAsString(rbm2) - expect2 := "c(1048576, 1048577)" - if s2 != expect2 { - panic(fmt.Sprintf("s2='%v', but expected '%v'", s2, expect2)) - } - - // now offset by 2M - offset = uint64(2 << 20) - tx3, _ := dbwrap.NewTx(!writable, index, Txo{}) - rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx) - PanicOn(err) - tx3.Rollback() - - //expect to see 3M == 3145728 - s3 := BitmapAsString(rbm3) - expect3 := "c(3145728, 3145729)" - - if s3 != expect3 { - panic(fmt.Sprintf("s3='%v', but expected '%v'", s3, expect3)) - } -} - -func TestBolt_Count_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Count_on_many_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{0, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - n, err := tx.Count(index, field, view, shard) - PanicOn(err) - if int(n) != len(putmeValues) { - panic(fmt.Sprintf("expected Count of %v but got n=%v", len(putmeValues), n)) - } -} - -func TestBolt_Count_dense_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Count_dense_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - expected := 0 - for i := uint64(0); i < (1<<16)+2; i += 2 { - changed, err := tx.Add(index, field, view, shard, i) - PanicOn(err) - if changed <= 0 { - panic("wat? should have changed") - } - expected++ - } - defer tx.Rollback() - - n, err := tx.Count(index, field, view, shard) - PanicOn(err) - if int(n) != expected { - panic(fmt.Sprintf("expected Count of %v but got n=%v", expected, n)) - } -} - -func TestBolt_ContainerIterator_on_empty(t *testing.T) { - // iterate on empty container, should not find anything. - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - bitvalue := uint64(0) - citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue) - PanicOn(err) - defer citer.Close() - if found { - panic("should not have found anything") - } - PanicOn(err) -} - -func TestBolt_ContainerIterator_on_one_bit(t *testing.T) { - // set one bit, iterate. - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_on_one_bit") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - bitvalue := uint64(42) - - // add a bit - changed, err := tx.Add(index, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - - // same Tx, continues in use. - - citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(bitvalue)) - if !found { - panic("ContainerIterator did not find the 42 bit") - } - PanicOn(err) - defer citer.Close() - - loopCount := 0 - for citer.Next() { - key, container := citer.Value() - if key != 0 { - panic("42 should have had key 0") - } - if container == nil { - panic("container was nil") - } - if container.N() != 1 { - panic("put a bit in, but size of container was not 1") - } - if !container.Contains(lowbits(bitvalue)) { - panic("container did not have our bitvalue!") - } - loopCount++ - if loopCount > 0 { // happier linter - break - } - } - if loopCount != 1 { - panic("ContainerIterator did not return a citer that scanned our set bit") - } -} - -func TestBolt_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_on_one_bit_fail_to_find") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - putme := uint64(1<<16) + 3 // in the key:1 container - searchme := putme + 1 - - // add a bit - changed, err := tx.Add(index, field, view, shard, putme) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic("ARG putme was NOT SET!!!") - } - - // same Tx, continues in use. - - citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(searchme)) - if !found { - panic("ContainerIterator did not find the searchme") - } - defer citer.Close() - loopCount := 0 - for citer.Next() { - key, container := citer.Value() - if key != 1 { - panic("Containeriterator searching for highbits(searchme) should not have had a bit") - } - if container == nil { - panic("container was nil") - } - if container.N() != 1 { - panic("put a bit in, but size of container was not 1") - } - if container.Contains(lowbits(searchme)) { - panic("container should have putme but not our searchme!") - } - loopCount++ - // only want first pass. keep linter happy by avoiding raw break - if loopCount > 0 { - break - } - } - PanicOn(err) -} - -func TestBolt_ContainerIterator_empty_iteration_loop(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_empty_iteration_loop") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - putme := uint64(1<<16) + 3 // in the key:1 container - searchme := uint64(1 << 17) // in the next container, key:2 - - // add a bit - changed, err := tx.Add(index, field, view, shard, putme) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic("ARG putme was NOT SET!!!") - } - - // same Tx, continues in use. - - citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(searchme)) - PanicOn(err) - if found { - panic("ContainerIterator found the searchme, when it should not have") - } - defer citer.Close() - if citer.Next() { - panic("expected no looping, 0 iterations, b/c started searchme past our data in putme") - } - - // expect to see a blow up from the citer.Value() call, verify that we do. - func() { - defer func() { - r := recover() - if r == nil { - panic("expected a panic from citer.Value() in this case") - } - }() - citer.Value() // should panic - }() - -} - -func TestBolt_ForEach_on_one_bit(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ForEach_on_one_bit") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - bitvalue := uint64(42) - - // add a bit - changed, err := tx.Add(index, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - - // same Tx, continues in use. - count := 0 - err = tx.ForEach(index, field, view, shard, func(v uint64) error { - if v != bitvalue { - panic(fmt.Sprintf("bitvalue corrupt got %v want %v", v, bitvalue)) - } - count += 1 - return nil - }) - PanicOn(err) - if count != 1 { - panic(fmt.Sprintf("Expected single iteration got %v ", count)) - } -} - -func TestBolt_RemoveContainer_one_bit_test(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_RemoveContainer_one_bit_test") - defer clean() - defer dbwrap.Close() - - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{0, 13, 77, 1511} - - for _, putme := range putmeValues { - - // a) delete of whole container in a seperate txn. Commit should establish the deletion. - - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // b) deletion + rollback on the txn should restore the deleted bit - - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // delete, but rollback instead of commit - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - hi := highbits(putme) - PanicOn(tx.RemoveContainer(index, field, view, shard, hi)) - tx.Rollback() - - // verify that the rollback undid the deletion. - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // c) within one Tx, after delete it should be gone as viewed within the txn. - tx, _ = dbwrap.NewTx(writable, index, Txo{}) - hi = highbits(putme) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme)) - } - - PanicOn(tx.RemoveContainer(index, field, view, shard, hi)) - - exists, err = tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if exists { - panic(fmt.Sprintf("ARG putme '%v' was SET even after RemoveContiner in this txn.", putme)) - } - - tx.Rollback() - - // verify that the rollback undid the deletion. - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - // leave with clean slate - BoltMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - } -} - -func TestBolt_Remove_one_bit_test(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Remove_one_bit_test") - defer clean() - defer dbwrap.Close() - - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{0, 13, 77, 1511} - - for _, putme := range putmeValues { - - // a) delete of whole container in a seperate txn. Commit should establish the deletion. - - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustDeleteBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // b) deletion + rollback on the txn should restore the deleted bit - - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // delete, but rollback instead of commit - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - hi, lo := highbits(putme), lowbits(putme) - _, _ = hi, lo - _, err := tx.Remove(index, field, view, shard, hi) - PanicOn(err) - tx.Rollback() - - // verify that the rollback undid the deletion. - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // c) within one Tx, after delete it should be gone as viewed within the txn. - tx, _ = dbwrap.NewTx(writable, index, Txo{}) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme)) - } - - mustRemove(tx.Remove(index, field, view, shard, putme)) - - exists, err = tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if exists { - panic(fmt.Sprintf("ARG putme '%v' was SET even after Remove in this txn.", putme)) - } - - tx.Rollback() - - // verify that the rollback undid the deletion. - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - // leave with clean slate - BoltMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - } -} - -func TestBolt_Min_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Min_on_many_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - // verify no containers flag works - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - min, containersExist, err := tx.Min(index, field, view, shard) - _ = min - PanicOn(err) - if containersExist { - panic("no containers should exist") - } - tx.Rollback() - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ = dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - min, containersExist, err = tx.Min(index, field, view, shard) - PanicOn(err) - if !containersExist { - panic("containers should exist") - } - expected := putmeValues[0] - if min != expected { - panic(fmt.Sprintf("expected Min() of %v but got min=%v", expected, min)) - } -} - -func TestBolt_CountRange_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_CountRange_on_many_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - // verify no containers flag works - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64) - PanicOn(err) - if n != 0 { - panic("no containers should exist") - } - tx.Rollback() - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ = dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64) - PanicOn(err) - if n == 0 { - panic("containers should exist") - } - expected := uint64(len(putmeValues)) - if n != expected { - panic(fmt.Sprintf("expected CountRange() of %v but got n=%v", expected, n)) - } -} - -func TestBolt_CountRange_middle_container(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_CountRange_middle_container") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - // pick out just the middle container with the 1 bit set on it. - n, err := tx.CountRange(index, field, view, shard, 4, (2<<16)+1) - PanicOn(err) - if n != 1 { - panic("middle 1 bit container should exist") - } -} - -func TestBolt_CountRange_many_middle_container(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_CountRange_many_middle_container") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - // get them all - n, err := tx.CountRange(index, field, view, shard, 0, (4<<16)+1) - PanicOn(err) - if n != 3 { - panic("count should have been all 3 bits") - } -} - -func TestBolt_UnionInPlace(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_UnionInPlace") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{3, 2 << 16} - - others := roaring.NewBitmap() - others2 := roaring.NewBitmap() - others3 := roaring.NewBitmap() - // populate others with putmeValues +1 into others - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) - n, err := tx2.Count(index, field, view, shard) - PanicOn(err) - if n != 2 { - panic("should have 2 bits set") - } - tx2.Rollback() - - for _, putme := range putmeValues { - mustAddR(others.Add(putme)) // should not change count, b/c putme already in the rbm - mustAddR(others.Add(putme + 1)) - mustAddR(others2.Add(putme + 2)) - } - mustAddR(others3.Add(4 << 16)) // outside the 2<<16 container - - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - err = tx.UnionInPlace(index, field, view, shard, others, others2, others3) - PanicOn(err) - - // end game, check we got the union. - rbm, err := tx.RoaringBitmap(index, field, view, shard) - PanicOn(err) - n = rbm.Count() - if n != 7 { - panic("should have a total 3 + 3 +1 = 7 bits set on the containers") - } -} - -func TestBolt_RoaringBitmap(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_RoaringBitmap") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - expected := uint64(3) - putme := expected - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - PanicOn(err) - - slc := rbm.Slice() - if slc[0] != uint64(expected) { - panic(fmt.Sprintf("should have gotten %v back", expected)) - } -} - -func TestBolt_ImportRoaringBits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ImportRoaringBits") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - tx.(*BoltTx).DeleteEmptyContainer = true // traditional lmdb Tx behavior, but not Roaring. - - //bitvalue := uint64(42) - - // get some roaring bits, get an itr RoaringIterator from them - rowSize := uint64(0) - //bits := []uint64{0} - bits := []uint64{0, 2, 5, 1<<16 + 1, 2 << 16} - data := getTestBitmapAsRawRoaring(bits...) - itr, err := roaring.NewRoaringIterator(data) - PanicOn(err) - clear := false - logme := false - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != len(bits) { - panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now test the union in place with the same set gives no change. - - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != 0 { - panic(fmt.Sprintf("should have not changed any bits on the second import, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now test the clear path - clear = true - - for _, v := range bits { - // clear 1 bit at a time - data := getTestBitmapAsRawRoaring(v) - itr, err := roaring.NewRoaringIterator(data) - PanicOn(err) - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != 1 { - panic(fmt.Sprintf("should have changed 1 bit: '%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - PanicOn(err) - } - n, err := tx.Count(index, field, view, shard) - PanicOn(err) - if n != 0 { - panic(fmt.Sprintf("n = %v not zero so the clearbits didn't happen!", n)) - } - allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) - - // should have no keys - if allkeys != "" { - panic("bolt should have no keys now") - } -} - -func TestBolt_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ImportRoaringBits_set_nonoverlapping_bits") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - // get some roaring bits, get an itr RoaringIterator from them - rowSize := uint64(0) - //bits := []uint64{0} - bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} - data := getTestBitmapAsRawRoaring(bits...) - itr, err := roaring.NewRoaringIterator(data) - PanicOn(err) - - bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16} - data2 := getTestBitmapAsRawRoaring(bits2...) - itr2, err := roaring.NewRoaringIterator(data2) - PanicOn(err) - - clear := false - logme := false - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != len(bits) { - panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now import the 2nd, overlapping set and set them. - - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil) - _ = rowSet - if changed != 4 { - panic(fmt.Sprintf("should have changed 2 bits: the 1 and the 3, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - PanicOn(err) -} - -func TestBolt_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ImportRoaringBits_clear_nonoverlapping_bits") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - // get some roaring bits, get an itr RoaringIterator from them - rowSize := uint64(0) - //bits := []uint64{0} - bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} //, 5, 1<<16 + 1, 2 << 16} - data := getTestBitmapAsRawRoaring(bits...) - itr, err := roaring.NewRoaringIterator(data) - PanicOn(err) - - bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16} - data2 := getTestBitmapAsRawRoaring(bits2...) - itr2, err := roaring.NewRoaringIterator(data2) - PanicOn(err) - - clear := false - logme := false - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != len(bits) { - panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now import the 2nd overlapping set and clear them. - clear = true - - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil) - _ = rowSet - if changed != 2 { - panic(fmt.Sprintf("should have changed 1 bit: the 2, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - PanicOn(err) - - n, err := tx.Count(index, field, view, shard) - PanicOn(err) - if n != 2 { // just the 0 and the 1<<16 bits should be left set. - panic(fmt.Sprintf("n = %v not 2 so the clearbits didn't happen!", n)) - } - -} - -func TestBolt_DeleteIndex(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_DeleteIndex") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - bitvalue := uint64(777) - bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} - for _, v := range bits { - changed, err := tx.Add(index, field, view, shard, v) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - } - - index2 := "i2" // should not be deleted, even though it shares a prefix with 'i' - changed, err := tx.Add(index2, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - } - exists, err := tx.Contains(index2, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!! on index2") - } - err = tx.Commit() - PanicOn(err) - - // end of setup - err = dbwrap.DeleteIndex(index) - PanicOn(err) - - tx, _ = dbwrap.NewTx(!writable, index2, Txo{}) - defer tx.Rollback() - exists, err = tx.Contains(index2, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2)) - } - - for _, v := range bits { - exists, err = tx.Contains(index, field, view, shard, v) - PanicOn(err) - if exists { - allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) - panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) - } - } -} - -func TestBolt_DeleteIndex_over100k(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_DeleteIndex_over100k") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - bitvalue := uint64(777) - limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction. - //limit := uint64(101) - for v := uint64(1); v < limit; v++ { - // shift by << 16 to get into a different shard - changed, err := tx.Add(index, field, view, shard, v<<16) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - if v%100000 == 0 { - PanicOn(tx.Commit()) - tx, _ = dbwrap.NewTx(writable, index, Txo{}) - } - } - - index2 := "i2" // should not be deleted, even though it shares a prefix with 'i' - changed, err := tx.Add(index2, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - err = tx.Commit() - PanicOn(err) - - // end of setup - err = dbwrap.DeleteIndex(index) - PanicOn(err) - - tx, _ = dbwrap.NewTx(!writable, index2, Txo{}) - defer tx.Rollback() - exists, err := tx.Contains(index2, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2)) - } - - for v := uint64(0); v < limit; v++ { - exists, err = tx.Contains(index, field, view, shard, v<<16) - PanicOn(err) - if exists { - allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) - panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) - } - } -} - -func TestBolt_HasData(t *testing.T) { - - db, clean := mustOpenEmptyBoltWrapper("TestBolt_SliceOfShards") - defer clean() - defer db.Close() - - // HasData should start out false. - hasAnything, err := db.HasData() - if err != nil { - t.Fatal(err) - } - if hasAnything { - t.Fatalf("HasData reported existing data on an empty database") - } - - // check that HasData sees a committed record. - - index, field, view, shard, putme := "i", "f", "v", uint64(123), uint64(42) - BoltMustSetBitvalue(db, index, field, view, shard, putme) - - // HasData(false) should now report data - hasAnything, err = db.HasData() - if err != nil { - t.Fatal(err) - } - if !hasAnything { - t.Fatalf("HasData() reported no data on a database that has bits written to it") - } -} diff --git a/catcher.go b/catcher.go index b807b8ea9..92ce5b2b9 100644 --- a/catcher.go +++ b/catcher.go @@ -15,9 +15,6 @@ package pilosa import ( - "fmt" - "io" - "github.com/molecula/featurebase/v2/roaring" txkey "github.com/molecula/featurebase/v2/short_txkey" . "github.com/molecula/featurebase/v2/vprint" @@ -42,40 +39,18 @@ func init() { var _ Tx = (*catcherTx)(nil) -func (c *catcherTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - c.b.IncrementOpN(index, field, view, shard, changedN) -} - func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { return c.b.NewTxIterator(index, field, view, shard) } -func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { +func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { defer func() { if r := recover(); r != nil { AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack()) PanicOn(r) } }() - return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) -} - -func (c *catcherTx) Dump(short bool, shard uint64) { - c.b.Dump(short, shard) -} - -func (c *catcherTx) Readonly() bool { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Readonly() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.Readonly() -} - -func (tx *catcherTx) Pointer() string { - return fmt.Sprintf("%p", tx) + return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize) } func (c *catcherTx) Rollback() { @@ -143,14 +118,6 @@ func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key return c.b.RemoveContainer(index, field, view, shard, key) } -func (c *catcherTx) UseRowCache() bool { - return c.b.UseRowCache() -} - -func (c *catcherTx) IsDone() bool { - return c.b.IsDone() -} - func (c *catcherTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { defer func() { @@ -250,17 +217,6 @@ func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, return c.b.Min(index, field, view, shard) } -func (c *catcherTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.UnionInPlace(index, field, view, shard, others...) -} - func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { defer func() { @@ -283,33 +239,10 @@ func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, return c.b.OffsetRange(index, field, view, shard, offset, start, end) } -func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) -} - func (c *catcherTx) Type() string { return c.b.Type() } -func (c *catcherTx) Group() *TxGroup { - return c.b.Group() -} - -func (c *catcherTx) Options() Txo { - return c.b.Options() -} - -// Sn retreives the serial number of the Tx. -func (c *catcherTx) Sn() int64 { - return c.b.Sn() -} - func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { return GenericApplyFilter(c, index, field, view, shard, ckey, filter) } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index cc55ae9aa..3d204ae92 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -17,7 +17,6 @@ package pilosa import ( "fmt" "math/rand" - "net" "reflect" "strings" "testing" @@ -30,57 +29,8 @@ import ( "github.com/molecula/featurebase/v2/testhook" "github.com/molecula/featurebase/v2/topology" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck - "github.com/pkg/errors" ) -// GlobalPortMap avoids many races and port conflicts when setting -// up ports for test clusters. Used for tests only. -var globalPortMap *GlobalPortMapper - -func init() { - globalPortMap = NewGlobalPortMapper(300) -} - -// GlobalPortMapper maintains a pool of available ports by -// holding them open until GetPort() is called. -type GlobalPortMapper struct { - availPorts map[int]net.Listener -} - -// reserve n ports -func NewGlobalPortMapper(n int) (pm *GlobalPortMapper) { - - pm = &GlobalPortMapper{ - availPorts: make(map[int]net.Listener), - } - for i := 0; i < n; i++ { - lsn, err := net.Listen("tcp", ":0") - if err != nil { - panic(errors.Wrap(err, "trying to listen on ephemeral port")) - } - r := lsn.Addr() - port := r.(*net.TCPAddr).Port - pm.availPorts[port] = lsn - } - return -} - -func (pm *GlobalPortMapper) GetPort() (port int, err error) { - for port, lsn := range pm.availPorts { - lsn.Close() - return port, nil - } - return -1, fmt.Errorf("no more ports available") -} - -func (pm *GlobalPortMapper) MustGetPort() int { - port, err := pm.GetPort() - if err != nil { - panic(err) - } - return port -} - // Ensure that fragCombos creates the correct fragment mapping. func TestFragCombos(t *testing.T) { uri0, err := pnet.NewURIFromAddress("host0") diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index 8b5b62d6c..627d56ef2 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -310,7 +310,7 @@ func Migrate(dataDir, backupPath string) error { return err } key := string(txkey.Prefix(index, field, view, shard)) - _, _, err = tx.ImportRoaringBits(key, itr, clear, log, rowSize, nil) + _, _, err = tx.ImportRoaringBits(key, itr, clear, log, rowSize) if err != nil { tx.Rollback() return err diff --git a/ctl/server.go b/ctl/server.go index 6c4405135..71c26bb3c 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -94,7 +94,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // over-ride. // TODO: the comment above was carried over from the PILOSA_TXSRC flag, but // we should confirm that this still applies. - flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend)) + flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring or rbf. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend)) flags.BoolVar(&srv.Config.Storage.FsyncEnabled, "storage.fsync", true, "enable fsync fully safe flush-to-disk") // RowcacheOn diff --git a/dbshard.go b/dbshard.go index a86cc4ab7..15bd41915 100644 --- a/dbshard.go +++ b/dbshard.go @@ -57,12 +57,10 @@ type DBIndex struct { type DBWrapper interface { NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) - DeleteDBPath(dbs *DBShard) error Close() error DeleteFragment(index, field, view string, shard uint64, frag interface{}) error DeleteField(index, field, fieldPath string) error OpenListString() string - OpenSnList() (sns []int64) Path() string HasData() (has bool, err error) SetHolder(h *Holder) @@ -82,134 +80,52 @@ type DBShard struct { Shard uint64 Open bool - // With RWMutex, the blue-green Tx can start and commit - // atomically. - mut sync.RWMutex - - types []txtype - stypes []string + typ txtype + styp string hasRoaring bool // if either of the types is roaringTxn - W []DBWrapper + W DBWrapper ParentDBIndex *DBIndex idx *Index per *DBPerShard - useOpenList int - closed bool - - isBlueGreen bool + closed bool } func (dbs *DBShard) DeleteFragment(index, field, view string, shard uint64, frag interface{}) (err error) { - for _, w := range dbs.W { - err = w.DeleteFragment(index, field, view, shard, frag) - if err != nil { - return err - } + if index != dbs.Index { + return fmt.Errorf("DeleteFragment called on DBShard for %q with index %q", dbs.Index, index) } - return + if shard != dbs.Shard { + return fmt.Errorf("DeleteFragment called on DBShard for %d with shard %d", dbs.Shard, shard) + } + return dbs.W.DeleteFragment(index, field, view, shard, frag) } func (dbs *DBShard) DeleteFieldFromStore(index, field, fieldPath string) (err error) { - for _, w := range dbs.W { - err = w.DeleteField(index, field, fieldPath) - if err != nil { - return err - } + if index != dbs.Index { + return fmt.Errorf("DeleteFieldFromStore called on DBShard for %q with index %q", dbs.Index, index) } - return + return dbs.W.DeleteField(index, field, fieldPath) } func (dbs *DBShard) Close() (err error) { - for _, w := range dbs.W { - err = w.Close() - if err != nil { - return err - } - } dbs.closed = true - return -} - -// Cleanup must be called at every commit/rollback of a Tx, in -// order to release the read-write mutex that guarantees a single -// writer at a time. Each tx must take care to call cleanup() -// exactly once. examples: -// tx.o.dbs.Cleanup(tx) -// tx.Options().dbs.Cleanup(tx) -// -func (dbs *DBShard) Cleanup(tx Tx) { - if dbs == nil { - return // some tests are using Tx only, no dbs available. - } - //vv("gid %v top of DBShard %v Cleanup for tx.Sn = %v; dbs=%p; is 2nd: %v; type='%v'; dbs.stypes='%#v'", curGID(), dbs.Shard, tx.Sn(), dbs, tx.Type() == dbs.stypes[1], tx.Type(), dbs.stypes) - if !dbs.hasRoaring { - if dbs.isBlueGreen { - // only release on the 2nd Tx's cleanup - if tx.Type() == dbs.stypes[1] { - if tx.Readonly() { - dbs.mut.RUnlock() - //vv("gid %v released read-lock on shard %v", curGID(), dbs.Shard) - } else { - dbs.mut.Unlock() - //vv("gid %v released write-lock on shard %v", curGID(), dbs.Shard) - } - } - } - } + return dbs.W.Close() } func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { - - if dbs.isBlueGreen { - // enforce only one writer at a time. The dbs.mut is held until - // the Tx finishes. This makes the two Tx in the blue-green Tx atomic. - if !dbs.hasRoaring { - if write { - //vv("shard %v about to write lock by gid %v; stack =\n%v", dbs.Shard, curGID(), stack()) - dbs.mut.Lock() - //vv("shard %v was write locked by gid %v; stack =\n%v", dbs.Shard, curGID(), stack()) - } else { - //vv("shard %v about to be read locked by gid %v; stack=\n%v", dbs.Shard, curGID(), stack()) - dbs.mut.RLock() - //vv("shard %v was read locked by gid %v; stack=\n%v", dbs.Shard, curGID(), stack()) - } - } + if initialIndexName != dbs.Index { + return nil, fmt.Errorf("NewTx called on DBShard for %q with index %q", dbs.Index, initialIndexName) } if o.dbs != dbs { - PanicOn(fmt.Sprintf("TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs)) + return nil, fmt.Errorf("dbs mismatch: TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs) } if o.Shard != dbs.Shard { - PanicOn(fmt.Sprintf("shard disagreement! o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard))) + return nil, fmt.Errorf("shard disagreement: o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard)) } - var txns []Tx - - for _, w := range dbs.W { - tx, err = w.NewTx(write, initialIndexName, o) - if err != nil { - return nil, err - } - txns = append(txns, tx) - } - if len(txns) == 1 { - return - } - // blue green - tx, err = dbs.per.txf.newBlueGreenTx(txns[0], txns[1], o.Index, o), nil - //vv("dbshard returning blue-green tx sn %v", tx.Sn()) - return -} - -func (dbs *DBShard) DeleteDBPath() (err error) { - for _, w := range dbs.W { - err = w.DeleteDBPath(dbs) - if err != nil { - return err - } - } - return + return dbs.W.NewTx(write, initialIndexName, o) } type flatkey struct { @@ -228,34 +144,26 @@ type DBPerShard struct { // Easily see how many we have. Flatmap map[flatkey]*DBShard - types []txtype + typ txtype hasRoaring bool txf *TxFactory holder *Holder - // which of our types is not-roaring, since - // roaring doesn't keep a list of open Tx sn. - // or default to the 2nd. - useOpenList int - // cache the shards per index to avoid excessive - // directory scans of the index directory. Keep per - // txtype to allow blue-green migrate open to be fast too. + // directory scans of the index directory. // Keep it up-to-date as we add shards to avoid doing // a filesystem rescan on new shard creation. // - // txtype -> index -> *shardSet - index2shards map[txtype]map[string]*shardSet - - isBlueGreen bool + // index -> *shardSet + index2shards map[string]*shardSet StorageConfig *storage.Config RBFConfig *rbfcfg.Config } -func newIndex2Shards() (r map[txtype]map[string]*shardSet) { - r = make(map[txtype]map[string]*shardSet) +func newIndex2Shards() (r map[string]*shardSet) { + r = make(map[string]*shardSet) return } @@ -350,51 +258,6 @@ func newShardSetFromMap(m map[uint64]bool) *shardSet { } } -// HasData returns true if the database has at least one key. -// For roaring it returns true if we a fragment stored. -// The `which` argument is the index into the per.W slice. 0 for blue, 1 for green. -// If you pass 1, be sure you have a blue-green configuration. -func (per *DBPerShard) HasData(which int) (hasData bool, err error) { - // has to aggregate across all available DBShard for each index and shard. - - if per.types[which] == roaringTxn { - return per.RoaringHasData() // this needs to be made accurate - } - - for _, v := range per.Flatmap { - hasData, err = v.W[which].HasData() - if err != nil { - return - } - if hasData { - return - } - } - return -} - -func (per *DBPerShard) RoaringHasData() (bool, error) { - idxs := per.holder.Indexes() - const requireData = true - for _, idx := range idxs { - shards, err := per.TypedDBPerShardGetShardsForIndex(roaringTxn, idx, "", requireData) - if err != nil { - return false, err - } - if len(shards) > 0 { - return true, nil - } - } - return false, nil -} - -func (per *DBPerShard) ListOpenString() (r string) { - for _, v := range per.Flatmap { - r += v.HolderPath + " -> " + v.W[per.useOpenList].OpenListString() + "\n" - } - return -} - func (per *DBPerShard) LoadExistingDBs() (err error) { idxs := per.holder.Indexes() @@ -414,37 +277,24 @@ func (per *DBPerShard) LoadExistingDBs() (err error) { return } -func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string, holder *Holder) (d *DBPerShard) { +func (txf *TxFactory) NewDBPerShard(typ txtype, holderDir string, holder *Holder) (d *DBPerShard) { if holder.cfg == nil || holder.cfg.RBFConfig == nil || holder.cfg.StorageConfig == nil { PanicOn("must have holder.cfg.RBFConfig and holder.cfg.StorageConfig set here") } - useOpenList := 0 hasRoaring := false - if types[0] == roaringTxn { + if typ == roaringTxn { hasRoaring = true } - if len(types) == 2 { - // blue-green, avoid the empty roaring Tx open list. - // Prefer B's open list if neither is roaring. - if types[0] == roaringTxn || types[1] != roaringTxn { - useOpenList = 1 - } - if types[1] == roaringTxn { - hasRoaring = true - } - } d = &DBPerShard{ - types: types, + typ: typ, HolderDir: holderDir, holder: holder, dbh: NewDBHolder(), Flatmap: make(map[flatkey]*DBShard), txf: txf, - useOpenList: useOpenList, hasRoaring: hasRoaring, - isBlueGreen: len(types) > 1, index2shards: newIndex2Shards(), StorageConfig: holder.cfg.StorageConfig, RBFConfig: holder.cfg.RBFConfig, @@ -469,14 +319,12 @@ func (per *DBPerShard) DeleteIndex(index string) (err error) { if err != nil { return errors.Wrap(err, "DBPerShard.DeleteIndex dbs.Close()") } - for _, ty := range per.types { - path := dbs.pathForType(ty) - err = os.RemoveAll(path) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("DBPerShard.DeleteIndex os.RemoveAll('%v')", path)) - } - delete(per.index2shards[ty], index) + path := dbs.pathForType(per.typ) + err = os.RemoveAll(path) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("DBPerShard.DeleteIndex os.RemoveAll('%v')", path)) } + delete(per.index2shards, index) } // allow the index to be created again anew. @@ -502,10 +350,8 @@ func (per *DBPerShard) DeleteFieldFromStore(index, field, fieldPath string) (err return nil } for _, dbs := range dbi.Shard { - for _, w := range dbs.W { - if e := w.DeleteField(index, field, fieldPath); e != nil && err == nil { - err = errors.Wrap(e, "DeleteFieldFromStore()") - } + if e := dbs.W.DeleteField(index, field, fieldPath); e != nil && err == nil { + err = errors.Wrap(e, "DeleteFieldFromStore()") } } return err @@ -521,46 +367,6 @@ func (per *DBPerShard) DeleteFragment(index, field, view string, shard uint64, f return dbs.DeleteFragment(index, field, view, shard, frag) } -func (dbs *DBShard) DumpAll() { - short := false - fmt.Printf("\n============= begin DumpAll dbs=%p index='%v', shard=%v ========\n", dbs, dbs.Index, int(dbs.Shard)) - for i, ty := range dbs.types { - _ = i - tx, err := dbs.W[i].NewTx(!writable, "", Txo{Index: dbs.idx}) - PanicOn(err) - defer tx.Rollback() - fmt.Printf("\n============= dumping dbs.W[%v] %v ========\n", i, ty) - tx.Dump(short, dbs.Shard) - - switch ty { - case roaringTxn: - case rbfTxn: - case boltTxn: - default: - PanicOn(fmt.Sprintf("unknown txtyp: '%v'", ty)) - } - } - fmt.Printf("\n============= end of DumpAll index='%v', shard=%v ========\n", dbs.Index, int(dbs.Shard)) -} - -func (per *DBPerShard) DumpAll() { - per.Mu.Lock() - defer per.Mu.Unlock() - - found1 := false - for _, dbi := range per.dbh.Index { - for _, dbs := range dbi.Shard { - if dbs.Open { - found1 = true - dbs.DumpAll() - } - } - } - if !found1 { - AlwaysPrintf("DBPerShard.DumpAll() sees no databases. dir='%v'", per.HolderDir) - } -} - // if you know the shard, you can use this // pathForType and prefixForType must be kept in sync! func (dbs *DBShard) pathForType(ty txtype) string { @@ -570,11 +376,6 @@ func (dbs *DBShard) pathForType(ty txtype) string { // is a no-op anyhow. so doesn't need to be correct atm. path := dbs.HolderPath + sep + dbs.Index + sep + backendsDir + sep + ty.DirectoryName() + sep + fmt.Sprintf("shard.%04v", dbs.Shard) - if ty == boltTxn { - // special case: - // bolt doesn't use a directory like the others, just a direct path. - path += sep + "bolt.db" - } return path } @@ -593,23 +394,13 @@ var ErrNoData = fmt.Errorf("no data") // // Caller must hold per.Mu.Lock() already. func (per *DBPerShard) updateIndex2ShardCacheWithNewShard(dbs *DBShard) { - - for _, ty := range dbs.types { - mapIndex2shardSet, ok := per.index2shards[ty] - if !ok { - mapIndex2shardSet = make(map[string]*shardSet) - per.index2shards[ty] = mapIndex2shardSet - } - // INVAR: mapIndex2shardSet is good, but may be an empty map - - shardset, ok := mapIndex2shardSet[dbs.Index] - if !ok { - shardset = newShardSet() - mapIndex2shardSet[dbs.Index] = shardset - } - // INVAR: shardset is present, not nil; a map that can be added to. - shardset.add(dbs.Shard) + shardset, ok := per.index2shards[dbs.Index] + if !ok { + shardset = newShardSet() + per.index2shards[dbs.Index] = shardset } + // INVAR: shardset is present, not nil; a map that can be added to. + shardset.add(dbs.Shard) } func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *DBShard, err error) { @@ -629,76 +420,47 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In } dbs, ok = dbi.Shard[shard] if dbs != nil && dbs.closed { - if len(per.types) == 1 && per.types[0] == roaringTxn { - // roaring txn are nil/fake anyway. Don't freak out. - } else { - PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.types[0]='%v'; len(per.types)=%v", dbs, per.types[0], len(per.types))) + // roaring txn are nil/fake anyway. Don't freak out. + if per.typ != roaringTxn { + PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.typ='%v'", dbs, per.typ)) } } if !ok { dbs = &DBShard{ - types: per.types, + typ: per.typ, ParentDBIndex: dbi, Index: index, Shard: shard, HolderPath: per.HolderDir, idx: idx, per: per, - useOpenList: per.useOpenList, hasRoaring: per.hasRoaring, - isBlueGreen: len(per.types) > 1, } - dbs.stypes = make([]string, len(per.types)) - for i, ty := range per.types { - dbs.stypes[i] = ty.String() - } - + dbs.styp = per.typ.String() dbi.Shard[shard] = dbs per.updateIndex2ShardCacheWithNewShard(dbs) } if !dbs.Open { var registry DBRegistry - for _, ty := range dbs.types { - switch ty { - case roaringTxn: - registry = globalRoaringReg - case rbfTxn: - registry = globalRbfDBReg - registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig) - case boltTxn: - registry = globalBoltReg - default: - PanicOn(fmt.Sprintf("unknown txtyp: '%v'", ty)) - } - path := dbs.pathForType(ty) - w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.StorageConfig) - PanicOn(err) - h := idx.Holder() - w.SetHolder(h) - dbs.Open = true - if w != nil && len(dbs.W) == 0 { - per.Flatmap[flatkey{index: index, shard: shard}] = dbs - } - dbs.W = append(dbs.W, w) + switch dbs.typ { + case roaringTxn: + registry = globalRoaringReg + case rbfTxn: + registry = globalRbfDBReg + registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig) + default: + PanicOn(fmt.Sprintf("unknown txtyp: '%v'", dbs.typ)) } + path := dbs.pathForType(dbs.typ) + w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.StorageConfig) + PanicOn(err) + h := idx.Holder() + w.SetHolder(h) + dbs.Open = true + per.Flatmap[flatkey{index: index, shard: shard}] = dbs + dbs.W = w } - return -} - -func (per *DBPerShard) Del(dbs *DBShard) (err error) { - per.Mu.Lock() - defer per.Mu.Unlock() - - err = dbs.Close() - if err != nil { - return - } - PanicOn(dbs.DeleteDBPath()) - delete(per.Flatmap, flatkey{index: dbs.Index, shard: dbs.Shard}) - - // delete from the heirarchy - delete(dbs.ParentDBIndex.Shard, dbs.Shard) - return nil + return dbs, nil } func (per *DBPerShard) Close() (err error) { @@ -718,28 +480,7 @@ func (per *DBPerShard) Close() (err error) { // If requireData, we open the database and see that it has a key, rather // than assume that the database file presence is enough. func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requireData bool) (map[uint64]bool, error) { - - n := len(f.types) - if n != 1 && n != 2 { - PanicOn(fmt.Sprintf("internal error. only green or blue/green supported. we see types len %v", n)) - } - - var shards []map[uint64]bool - for _, ty := range f.types { - ss, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(ty, idx, roaringViewPath, requireData) - if err != nil { - return nil, err - } - shards = append(shards, ss) - } - - // Note: we don't actually know when the blue call and when the green call comes - // through here. So if we are deleting a shard, we will see a difference earlier - // in one than the other. TestAPI_ClearFlagForImportAndImportValues for example. - // Hence we cannot do a blue-green check here for matching shards. - - // If we are populating blue from green, it does matter that we return green. - return shards[n-1], nil + return f.dbPerShard.TypedDBPerShardGetShardsForIndex(f.typ, idx, roaringViewPath, requireData) } // if roaringViewPath is "" then for ty == roaringTxn we go to disk to discover @@ -751,10 +492,6 @@ func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requir // when a new DBShard is made, we will update the list of shards then. Thus // the per.index2shard should always be up to date AFTER the first call here. // -// Note: we cannot here call GetView2ShardsMapForIndex() because that only ever -// returns the green data and we are used during migration for both blue -// and green. -// func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, roaringViewPath string, requireData bool) (shardMap map[uint64]bool, err error) { // use the cache, always @@ -769,13 +506,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r return shardMap, nil } - i2ss, ok := per.index2shards[ty] - if !ok { - // index -> shardSet - i2ss = make(map[string]*shardSet) - per.index2shards[ty] = i2ss - } - // INVAR: i2ss is good, but may be an empty map + i2ss := per.index2shards ss, ok := i2ss[idx.name] if ok { @@ -785,7 +516,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r // gotta read shards from disk directory layout. setOfShards := newShardSet() - per.index2shards[ty][idx.name] = setOfShards + per.index2shards[idx.name] = setOfShards // Upon return, cache the setOfShards value and reuse it next time @@ -854,13 +585,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r } func (per *DBPerShard) unprotectedTypedIndexShardHasData(ty txtype, idx *Index, shard uint64) (hasData bool, err error) { - whichty := 0 - if len(per.types) == 2 { - if ty == per.types[1] { - whichty = 1 - } - } - if ty != per.types[whichty] { + if ty != per.typ { return } @@ -871,7 +596,7 @@ func (per *DBPerShard) unprotectedTypedIndexShardHasData(ty txtype, idx *Index, "per.GetDBShard(index='%v', shard='%v', ty='%v')", idx.name, shard, ty.String())) } - return dbs.W[whichty].HasData() + return dbs.W.HasData() } func listDirUnderDir(root string, includeRoot bool, ignoreEmpty bool) (files []string, err error) { @@ -906,224 +631,6 @@ func listDirUnderDir(root string, includeRoot bool, ignoreEmpty bool) (files []s return } -// populateBlueFromGreen prepares for a blue_green run at startup time. -// -// It is called at the end of Holder.Open(). This allows the application -// of blue-green checking to pilosa instances that -// were previously run only with a single (solo) backend. -// -// PRE: This operation requires, at its start, either: -// -// (1) an empty blue database -- this allows transitioning from -// a solo database to blue_green checking where the solo -// becomes the green; or -// -// (2) that the blue data, if present, be logically -// identical to the green data -- this allows one to restart -// a pilosa that was already running in blue_green mode -// and remain in blue_green mode. -// -// In either case, the goal to to finish populateBlueFromGreen() -// and have the exact same logical set of data in both backends. -// -// Why must the data be identical after Holder.Open() finishes? -// Otherwise subsequent blue-green checks have no hope of -// being accurate. -// -// The blue is the destination -- this is always types[0]. -// The green source is always types[1]. The mnemonic is blue_geen. -// The blue is first, so it is in types[0]. The green -// is second, in types[1]. For example, with PILOSA_STORAGE_BACKEND=bolt_roaring -// we have bolt as blue, and roaring as green. The contents of -// bolt must be empty or exactly match roaring. If bolt -// starts empty, it will be populated from roaring by -// populateBlueFromGreen(). -// -func (dbs *DBShard) populateBlueFromGreen() (err error) { - - n := len(dbs.W) - if n != 2 { - PanicOn(fmt.Sprintf("populateBlueFromGreen did not find 2 open DBs: have %v", n)) - } - - dest := dbs.W[0] // blue - src := dbs.W[1] // green - - // copy all the key/container pairs. - // Since a shard is fairly small, we think one Tx will suffice. - - readtx, err := src.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard}) - PanicOn(err) - defer readtx.Rollback() - - writetx, err := dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard}) - PanicOn(err) - defer writetx.Rollback() - - ctWriteCount := 0 - - for _, fld := range dbs.idx.Fields() { - field := fld.Name() - for _, vw := range fld.views() { - view := vw.name - citer, _, err := readtx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0) - if err != nil { - // might be an empty fragment. If so, let's not freak out. - if strings.Contains(err.Error(), "fragment not found") { - continue - } else { - writetx.Rollback() - return errors.Wrap(err, "DBShard.populateBlueFromGreen readtx.ContainerIterator") - } - } - - for citer.Next() { - ckey, rc := citer.Value() - err := writetx.PutContainer(dbs.Index, field, view, dbs.Shard, ckey, rc) - if err != nil { - citer.Close() - writetx.Rollback() - return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.PutContainer") - } - - ctWriteCount++ - if ctWriteCount%1000 == 1 { - - // regularly commiting smaller batches and the first batch as soon as - // possible massively speeds up writing to bolt. - // - // reference: https://github.com/boltdb/bolt/issues/94 - // - // benbjohnson commented on Mar 25, 2014 - // "Bulk loading more than 1000 items at a time is very slow. This is because nodes - // are not splitting before commit which causes large memmove() operations during insertion." - // runtime.memmove is taking all of the time in our pprof profile, when copying rbf to bolt, so we suspect it is this. - // - err = writetx.Commit() - if err != nil { - citer.Close() - writetx.Rollback() - return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.Commit") - } - writetx, err = dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard}) - if err != nil { - citer.Close() - writetx.Rollback() - return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.NewTx inside citer.Next() loop") - } - } - - } - citer.Close() - } - } - err = writetx.Commit() - if err != nil { - return errors.Wrap(err, "writetx.Commit()") - } - return nil -} - -// verifyBlueEqualsGreen checks that blue and green are identical. -func (dbs *DBShard) verifyBlueEqualsGreen() (err error) { - - n := len(dbs.W) - if n != 2 { - PanicOn(fmt.Sprintf("verifyBlueEqualsGreen did not find 2 open DBs: have %v", n)) - } - - blue := dbs.W[0] - green := dbs.W[1] - - greentx, err := green.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard}) - PanicOn(err) - defer greentx.Rollback() - - bluetx, err := blue.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard}) - PanicOn(err) - defer bluetx.Rollback() - - for _, fld := range dbs.idx.Fields() { - field := fld.Name() - for _, vw := range fld.views() { - - view := vw.name - gCiter, _, err := greentx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0) - if err != nil { - if strings.Contains(err.Error(), "fragment not found") { - continue - } else { - return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen greentx.ContainerIterator") - } - } - - bCiter, _, err := bluetx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0) - if err != nil { - gCiter.Close() - if bCiter != nil { - bCiter.Close() - } - return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen bluetx.ContainerIterator") - } - - for gCiter.Next() { - greenCkey, greenc := gCiter.Value() - - if !bCiter.Next() { - bCiter.Close() - gCiter.Close() - return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ - "sees missing blue container at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v' the greenCkey: '%v'", - dbs.Index, field, view, dbs.Shard, greenCkey)) - } - blueCkey, bluec := bCiter.Value() - - if blueCkey != greenCkey { - bCiter.Close() - gCiter.Close() - return fmt.Errorf("DBShard.verifyBlueEqualsGreen sees sequence-of-ckey "+ - "difference: blueCkey %v not equal to greenCkey %v at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v'", - blueCkey, greenCkey, dbs.Index, field, view, dbs.Shard) - } - nGreen := greenc.N() - nBlue := bluec.N() - if nBlue != nGreen { - bCiter.Close() - gCiter.Close() - return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ - "sees variation in blue at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v", - dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue)) - } - err = bluec.BitwiseCompare(greenc) - if err != nil { - bCiter.Close() - gCiter.Close() - return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ - "sees variation in blue at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v ; BitwiseCompare response: '%v'", - dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue, err)) - } - } - if bCiter.Next() { - blueCkey, _ := bCiter.Value() - bCiter.Close() - gCiter.Close() - return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ - "sees extra blue container (not present in green) at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v' the ckey: '%v'", - dbs.Index, field, view, dbs.Shard, blueCkey)) - } - bCiter.Close() - gCiter.Close() - } - } - - return nil -} - type FieldView2Shards struct { // field -> view -> *shardSet m map[string]map[string]*shardSet @@ -1232,15 +739,8 @@ func (vs *FieldView2Shards) removeField(name string) { delete(vs.m, name) } -// Note: cannot call this during migration, because -// it only ever returns the green shards if we are in blue-green. func (per *DBPerShard) GetFieldView2ShardsMapForIndex(idx *Index) (vs *FieldView2Shards, err error) { - - // for blue-green, it does matter that we return green, so we can migrate from it. - ty := per.types[0] - if per.isBlueGreen { - ty = per.types[1] - } + ty := per.typ switch ty { case roaringTxn: diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go index 9482d1504..13c561810 100644 --- a/dbshard_internal_test.go +++ b/dbshard_internal_test.go @@ -84,7 +84,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { v2s.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet) } - for _, src := range []string{"roaring", "bolt", "rbf"} { + for _, src := range []string{"roaring", "rbf"} { cfg := mustHolderConfig() cfg.StorageConfig.Backend = src holder := NewHolder(tmpdir, cfg) @@ -145,7 +145,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { tx.Rollback() } } else { - // non-roaring: rbf, bolt + // non-roaring: rbf for _, shard := range []uint64{93, 223, 221, 215, 219, 217} { tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard}) @@ -191,14 +191,6 @@ rick/fields/_exists/views/standard/fragments/217 rick/fields/_exists/views/standard/fragments/93 rick/fields/_exists/views/standard/fragments/219 rick/fields/_exists/views/standard/fragments/223 -`, - "bolt": ` -rick/backends/backend-boltdb/shard.0093-bolt/bolt.db -rick/backends/backend-boltdb/shard.0215-bolt/bolt.db -rick/backends/backend-boltdb/shard.0217-bolt/bolt.db -rick/backends/backend-boltdb/shard.0219-bolt/bolt.db -rick/backends/backend-boltdb/shard.0221-bolt/bolt.db -rick/backends/backend-boltdb/shard.0223-bolt/bolt.db `, "rbf": ` rick/backends/backend-rbf/shard.0093-rbf @@ -225,7 +217,7 @@ func makeSampleRoaringDir(t *testing.T, root, index, backend string, minBytes in } var shard uint64 switch backend { - case "bolt", "rbf": + case "rbf": shard = shards[i] idx = helperCreateDBShard(h, index, shard) @@ -266,16 +258,8 @@ func helperCreateDBShard(h *Holder, index string, shard uint64) *Index { } // keep the ocd linter happy -var _ = makeBolttestDB var _ = makeRBFtestDB -func makeBolttestDB(path string, h *Holder, shard uint64) { - i := uint64(1) - w, _ := mustOpenEmptyBoltWrapper(path) - BoltMustSetBitvalue(w, "index", "field", "view", shard, i) - w.Close() -} - func makeRBFtestDB(path string, h *Holder, shard uint64) { i := uint64(1) diff --git a/delete_test.go b/delete_test.go index 061b72d1d..886b71e81 100644 --- a/delete_test.go +++ b/delete_test.go @@ -27,7 +27,6 @@ import ( ) func TestExecutor_DeleteRecords(t *testing.T) { - pilosa.NotBlueGreenTest(t) indexName := "i" setup := func(t *testing.T, r *require.Assertions, c *test.Cluster) { t.Helper() diff --git a/executor_test.go b/executor_test.go index adcf64dbb..ba357d724 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3843,7 +3843,6 @@ func TestExecutor_Execute_Existence(t *testing.T) { hldr2 := c.GetHolder(0) index2 := hldr2.Index("i") _ = index2 - //index2.Dump("after reopen") if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { t.Fatal(err) diff --git a/fragment.go b/fragment.go index daab8bb34..d3c52c7dc 100644 --- a/fragment.go +++ b/fragment.go @@ -47,6 +47,7 @@ import ( "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/shardwidth" "github.com/molecula/featurebase/v2/stats" + "github.com/molecula/featurebase/v2/storage" "github.com/molecula/featurebase/v2/testhook" "github.com/molecula/featurebase/v2/topology" "github.com/molecula/featurebase/v2/tracing" @@ -75,9 +76,6 @@ const ( // snapshotExt is the file extension used for an in-process snapshot. snapshotExt = ".snapshotting" - // copyExt is the file extension used for the temp file used while copying. - copyExt = ".copying" - // cacheExt is the file extension for persisted cache ids. cacheExt = ".cache" @@ -441,7 +439,7 @@ func (f *fragment) inspectStorage(data []byte, file *os.File, newGen generation, // (remapping an existing bitmap to match a new backing store). func (f *fragment) openStorage(unmarshalData bool) error { - useRowCache := f.idx.Txf().UseRowCache() + useRowCache := storage.RowCacheEnabled() if !f.idx.NeedsSnapshot() { f.gen = &NopGeneration{} if useRowCache { @@ -626,7 +624,7 @@ func (f *fragment) mustRow(tx Tx, rowID uint64) *Row { // (updating the cache). func (f *fragment) unprotectedRow(tx Tx, rowID uint64) (*Row, error) { - useRowCache := tx.UseRowCache() + useRowCache := storage.RowCacheEnabled() if useRowCache { if f.rowCache == nil { f.rowCache = newSimpleCache() @@ -699,7 +697,7 @@ func (f *fragment) setBit(tx Tx, rowID, columnID uint64) (changed bool, err erro if tx.Type() == RoaringTxn { return changed, errors.New("internal error: f.gen was nil and tx.Type is RoaringTxn - should never happen under roaring b/c storage should be open") } - // else blue green or transactional backend. Just do it. + // else transactional backend. Just do it. err = doSetFunc() } return changed, err @@ -744,7 +742,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo delete(f.checksums, int(rowID/HashBlockSize)) // Increment number of operations until snapshot is required. - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, 1) + f.incrementOpN(1) // If we're using a cache, update it. Otherwise skip the // possibly-expensive count operation. @@ -757,7 +755,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo } // Drop the rowCache entry; it's wrong, and we don't want to force // a new copy if no one's reading it. - if tx.UseRowCache() && f.rowCache != nil { + if storage.RowCacheEnabled() && f.rowCache != nil { f.rowCache.Add(rowID, nil) } @@ -809,7 +807,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b delete(f.checksums, int(rowID/HashBlockSize)) // Increment number of operations until snapshot is required. - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, 1) + f.incrementOpN(1) // If we're using a cache, update it. Otherwise skip the // possibly-expensive count operation. @@ -822,7 +820,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b } // Drop the rowCache entry; it's wrong, and we don't want to force // a new copy if no one's reading it. - if tx.UseRowCache() && f.rowCache != nil { + if storage.RowCacheEnabled() && f.rowCache != nil { f.rowCache.Add(rowID, nil) } @@ -891,7 +889,7 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo } // invalidate rowCache for this row. - if tx.UseRowCache() && f.rowCache != nil { + if storage.RowCacheEnabled() && f.rowCache != nil { f.rowCache.Add(rowID, nil) } @@ -942,7 +940,7 @@ func (f *fragment) unprotectedClearRow(tx Tx, rowID uint64) (changed bool, err e // Clear the row in cache. f.cache.Add(rowID, 0) - if tx.UseRowCache() && f.rowCache != nil { + if storage.RowCacheEnabled() && f.rowCache != nil { f.rowCache.Add(rowID, nil) } @@ -2426,7 +2424,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64 if f.storage != nil { wp = &f.storage.OpWriter } - useRowCache := tx.UseRowCache() + useRowCache := storage.RowCacheEnabled() doFunc := func() error { if len(set) > 0 { @@ -2438,7 +2436,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64 return errors.Wrap(err, "adding positions") } f.stats.Count(MetricImportedN, int64(changedN), 1) - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, changedN) + f.incrementOpN(changedN) } if len(clear) > 0 { @@ -2448,7 +2446,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64 return errors.Wrap(err, "clearing positions") } f.stats.Count(MetricClearedN, int64(changedN), 1) - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, changedN) + f.incrementOpN(changedN) } // Update cache counts for all affected rows. @@ -2735,7 +2733,7 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b rowSize := uint64(1 << shardVsContainerExponent) span, ctx := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits") - useRowCache := tx.UseRowCache() + useRowCache := storage.RowCacheEnabled() var changed int var rowSet map[uint64]int var wp *io.Writer @@ -2749,7 +2747,7 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b return err } - changed, rowSet, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, rit, clear, true, rowSize, nil) + changed, rowSet, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, rit, clear, true, rowSize) return err }) @@ -2789,7 +2787,7 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b span, _ = tracing.StartSpanFromContext(ctx, "importRoaring.incrementOpN") - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, changed) + f.incrementOpN(changed) span.Finish() return nil @@ -2815,6 +2813,10 @@ func (f *fragment) incrementOpN(changed int) { if changed <= 0 { return } + // don't count opN or ops if our index doesn't want snapshots + if !f.idx.NeedsSnapshot() { + return + } f.opN += changed f.ops++ if f.opN > f.MaxOpN { @@ -3015,11 +3017,15 @@ func (f *fragment) writeStorageToArchive(tw *tar.Writer) error { tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Shard: f.shard}) defer tx.Rollback() - file, sz, err := tx.RoaringBitmapReader(f.index(), f.field(), f.view(), f.shard, f.path()) + rbm, err := tx.RoaringBitmap(f.index(), f.field(), f.view(), f.shard) if err != nil { - return err + return errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") + } + var buf bytes.Buffer + sz, err := rbm.WriteTo(&buf) + if err != nil { + return errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") } - defer file.Close() // Write archive header. if err := tw.WriteHeader(&tar.Header{ @@ -3033,7 +3039,7 @@ func (f *fragment) writeStorageToArchive(tw *tar.Writer) error { // Copy the file up to the last known size. // This is done outside the lock because the storage format is append-only. - if _, err := io.CopyN(tw, file, sz); err != nil { + if _, err := io.CopyN(tw, &buf, sz); err != nil { return errors.Wrap(err, "copying") } return nil @@ -3086,8 +3092,7 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) { // Process file based on file name. switch hdr.Name { case "data": - idx := f.holder.Index(f.index()) - tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx := f.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() if err := f.fillFragmentFromArchive(tx, tr); err != nil { return 0, errors.Wrap(err, "reading storage") @@ -3131,7 +3136,7 @@ func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error { if err != nil { return errors.Wrap(err, "fillFragmentFromArchive NewRoaringIterator") } - changed, rowSet, err := tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, clear, log, rowSize, data) + changed, rowSet, err := tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, clear, log, rowSize) _, _ = changed, rowSet if err != nil { return errors.Wrap(err, "fillFragmentFromArchive ImportRoaringBits") @@ -3139,40 +3144,6 @@ func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error { return nil } -func (f *fragment) readStorageFromArchive(r io.Reader) error { - - // Create a temporary file to copy into. - path := f.path() + copyExt - file, err := os.Create(path) - if err != nil { - return errors.Wrap(err, "creating directory") - } - defer file.Close() - - // Copy reader into temporary path. - if _, err = io.Copy(file, r); err != nil { - return errors.Wrap(err, "copying") - } - - // TODO(jea): isn't this next Rename a file handle leak? - // try closing first - if err := f.closeStorage(); err != nil { - return errors.Wrap(err, "closeStorage-prior-to-Rename-and-openStorage") - } - - // Move snapshot to data file location. - if err := os.Rename(path, f.path()); err != nil { - return errors.Wrap(err, "renaming") - } - - // Reopen storage. - if err := f.openStorage(true); err != nil { - return errors.Wrap(err, "opening") - } - - return nil -} - func (f *fragment) readCacheFromArchive(r io.Reader) error { // Slurp data from reader and write to disk. buf, err := ioutil.ReadAll(r) @@ -3369,7 +3340,7 @@ func (f *fragment) intRowIterator(tx Tx, wrap bool, filters ...roaring.BitmapFil // accumulator [column ID] -> [int value] acc := make(map[uint64]int64) - if tx.UseRowCache() { + if storage.RowCacheEnabled() { // needs a write lock since it will update the f.rowCache f.mu.Lock() defer f.mu.Unlock() diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 9b4ac461a..23c2063f1 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -183,7 +183,6 @@ func TestFragment_RowcacheMap(t *testing.T) { // Ensure a fragment can clear a row. func TestFragment_ClearRow(t *testing.T) { - NotBlueGreenTest(t) f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) @@ -215,7 +214,6 @@ func TestFragment_ClearRow(t *testing.T) { // Ensure a fragment can set a row. func TestFragment_SetRow(t *testing.T) { - NotBlueGreenTest(t) f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 7, "") _ = idx defer f.Clean(t) @@ -1728,7 +1726,7 @@ func roaringOnlyBenchmark(b *testing.B) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { - roaringOnlyTest(t) + // roaringOnlyTest(t) f0, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") defer f0.Clean(t) @@ -1741,6 +1739,10 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } else if _, err := f0.clearBit(tx, 1000, 1); err != nil { t.Fatal(err) } + err := tx.Commit() + if err != nil { + t.Fatalf("committing write: %v", err) + } // Verify cache is populated. if n := f0.cache.Len(); n != 1 { @@ -1755,7 +1757,9 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Read into another fragment. - f1, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") + f1, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") + tx.Rollback() + defer f1.Clean(t) if rn, err := f1.ReadFrom(&buf); err != nil { // eventually calls fragment.fillFragmentFromArchive @@ -1763,6 +1767,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } else if wn != rn { t.Fatalf("read/write byte count mismatch: wn=%d, rn=%d", wn, rn) } + // make a read-only Tx after ReadFrom has committed. + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f1, Shard: f1.shard}) // Verify cache is in other fragment. if n := f1.cache.Len(); n != 1 { @@ -3049,7 +3055,7 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { // to generate an op log and/or snapshot. itr, err := roaring.NewRoaringIterator(data) PanicOn(err) - _, _, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, false, false, 0, nil) + _, _, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, false, false, 0) if err != nil { b.Errorf("import error: %v", err) } @@ -5215,15 +5221,13 @@ func TestImportValueConcurrent(t *testing.T) { // we will be making a new Tx each time, so we can rollback the default provided one. tx.Rollback() - types := idx.holder.txf.TxTypes() - for _, ty := range types { - switch ty { - case roaringTxn: - t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " + - "blueGreenTx because the lack of transactional consistency " + - "from Roaring-per-file will create false comparison " + - "failures.")) - } + ty := idx.holder.txf.TxTyp() + switch ty { + case roaringTxn: + t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " + + "roaring because the lack of transactional consistency " + + "from Roaring-per-file will create false comparison " + + "failures.")) } eg := &errgroup.Group{} @@ -5364,11 +5368,6 @@ func TestImportValueRowCache(t *testing.T) { // do we see races/corruption around concurrent read/write. // especially on writes to the row cache. func TestFragmentConcurrentReadWrite(t *testing.T) { - // actual transaction backends, there won't be any - // data, and in particular, the blue-green tests will - // note this and fire a false-positive. - NotBlueGreenTest(t) - f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) tx.Rollback() @@ -5528,12 +5527,6 @@ func TestFragment_Bug_Q2DoubleDelete(t *testing.T) { } } -func NotBlueGreenTest(t *testing.T) { - if strings.Contains(CurrentBackend(), "_") { - t.Skip("skip under blue green") - } -} - var mutexSamplesPrepared sync.Once func requireMutexSampleData(tb testing.TB) { diff --git a/go.mod b/go.mod index ff7c1d715..67d9c30bd 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,6 @@ require ( github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect - github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 github.com/go-test/deep v1.0.7 github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.3.3 diff --git a/go.sum b/go.sum index 0959b010c..60db445b5 100644 --- a/go.sum +++ b/go.sum @@ -86,10 +86,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E= -github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= diff --git a/holder.go b/holder.go index dea18e3e7..756d75443 100644 --- a/holder.go +++ b/holder.go @@ -302,7 +302,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { txf, err := NewTxFactory(cfg.StorageConfig.Backend, h.IndexesPath(), h) PanicOn(err) h.txf = txf - h.txf.blueGreenOffIfRunningBlueGreen() _ = testhook.Created(h.Auditor, h, nil) return h @@ -605,8 +604,6 @@ func (h *Holder) Open() error { h.txf = txf } - h.txf.blueGreenOffIfRunningBlueGreen() - // Reset closing in case Holder is being reopened. h.closing = make(chan struct{}) @@ -713,13 +710,6 @@ func (h *Holder) Open() error { return errors.Wrap(err, "Holder.Open h.txf.Open()") } - // under blue_green, we must sync blue from green before we turn on checking. - if err := h.txf.green2blue(h); err != nil { - return errors.Wrap(err, "Holder.Open h.txf.green2blue(h)") - } - - h.txf.blueGreenOnIfRunningBlueGreen() - if h.cfg.LookupDBDSN != "" { h.Logger.Printf("connecting to lookup database") @@ -814,9 +804,6 @@ func (h *Holder) Close() error { if globalUseStatTx { fmt.Printf("%v\n", globalCallStats.report()) } - if h.txf != nil && h.txf.blueGreenReg != nil { - h.txf.blueGreenReg.Close() - } h.Stats.Close() @@ -2131,12 +2118,6 @@ func (h *Holder) addIndex(idx *Index) { h.imu.Unlock() } -func (h *Holder) DumpAllShards() { - h.mu.RLock() - defer h.mu.RUnlock() - h.txf.dbPerShard.DumpAll() -} - func (h *Holder) Txf() *TxFactory { h.mu.Lock() defer h.mu.Unlock() diff --git a/holder_internal_test.go b/holder_internal_test.go index bd35f0b03..7cd8ad630 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -22,7 +22,6 @@ import ( "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/testhook" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck ) var _ = fmt.Printf @@ -109,73 +108,6 @@ func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID ui } } -func testMustHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) { - - //shard := columnID / ShardWidth - - // hmm... if its a new holder, meta data isn't there, so ask for it. - idx, err := h.CreateIndexIfNotExists(index, IndexOptions{}) - PanicOn(err) - - f := idx.Field(field) - if f == nil { - t.Fatalf("no such field '%v'", field) - } - - row, err := f.Row(nil, rowID) - if err != nil { - t.Fatalf("error getting field.Row(rowID=%v): %v", rowID, err) - } - - cols := row.Columns() - if len(cols) == 0 { - t.Fatalf("error getting field.Row().Columns(): empty columns, colID %v bit was not hot", columnID) - } - - for _, c := range cols { - if c == columnID { - return // ok, found it. - } - } - t.Fatalf("error getting field.Row().Columns(): colID %v bit was not hot", columnID) -} - -func testMustNotHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) { - if testHasBit(t, h, index, field, rowID, columnID) { - t.Fatalf("error, expected no bit but this bit was hot: index='%v', field='%v', rowID='%v', columnID='%v'", index, field, rowID, columnID) - } -} - -func testHasBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) bool { - - idx := h.Index(index) - if idx == nil { - return false // not even an index by this name. Obviously no hot bits either. - } - - f := idx.Field(field) - if f == nil { - return false - } - - row, err := f.Row(nil, rowID) - if err != nil { - return false - } - - cols := row.Columns() - if len(cols) == 0 { - return false - } - - for _, c := range cols { - if c == columnID { - return true // ok, found it. - } - } - return false -} - func TestHolderOperatorProcess(t *testing.T) { h, path, err := makeHolder(t, "") if err != nil { diff --git a/index.go b/index.go index 4cfd301b2..8506184f5 100644 --- a/index.go +++ b/index.go @@ -27,7 +27,6 @@ import ( "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/stats" "github.com/molecula/featurebase/v2/testhook" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -464,7 +463,6 @@ func (i *Index) Close() error { // make it clear what the Index.AvailableShards() calls are trying to obtain. const includeRemote = false -const localOnly = true // AvailableShards returns a bitmap of all shards with data in the index. func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap { @@ -853,14 +851,6 @@ func FormatQualifiedIndexName(index string) string { return fmt.Sprintf("%s\x00", index) } -// Dump prints to stdout the contents of the roaring Containers -// stored in idx. Mostly for debugging. -func (i *Index) Dump(label string) { - fileline := FileLine(2) - fmt.Printf("\n%v Dump: %v\n\n", fileline, label) - i.holder.txf.dbPerShard.DumpAll() -} - func (i *Index) Txf() *TxFactory { return i.holder.txf } diff --git a/pjobs.go b/pjobs.go deleted file mode 100644 index 0636fbd36..000000000 --- a/pjobs.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "sync" - - "github.com/glycerine/idem" -) - -// parallelJobs runs functions in parallel on a goroutine -// pool that has nGoro goroutines. -type parallelJobs struct { - nGoro int - - jobQ chan func(worker int) error - halters []*idem.Halter - - // err is protected by errmu - err error - errmu sync.Mutex -} - -func newParallelJobs(nGoro int) (p *parallelJobs) { - if nGoro < 1 { - // 0 really means, - // "turn it up to 11". - // same for negative. - nGoro = 10000 - } - // maximum 10K goroutines - if nGoro > 10000 { - nGoro = 10000 - } - - p = ¶llelJobs{ - nGoro: nGoro, - jobQ: make(chan func(worker int) error, 10000), - halters: make([]*idem.Halter, nGoro), - } - - for j := 0; j < nGoro; j++ { - h := idem.NewHalter() - p.halters[j] = h - } - - for i, h := range p.halters { - go func(h *idem.Halter, worker int) { - defer h.MarkDone() - for { - select { - case <-h.ReqStop.Chan: - return - case f, ok := <-p.jobQ: - if !ok { - // channel closed, finish up - return - } - - err1 := f(worker) - if err1 != nil { - p.errmu.Lock() - if p.err == nil { - p.err = err1 - } - p.errmu.Unlock() - // an error occurred, tell everyone to stop - for _, h2 := range p.halters { - h2.RequestStop() - } - return - } - } - } - }(h, i) - } - return -} - -// return value accepted will be false if we are shutting down -// due to an error. -func (p *parallelJobs) run(fun func(worker int) error) (accepted bool) { - select { - case <-p.halters[0].ReqStop.Chan: - return false - case p.jobQ <- fun: - return true - } -} - -func (p *parallelJobs) waitForFinish() error { - - // tell the workers no more jobs. - close(p.jobQ) - - // wait for everyone to finish - for i, h := range p.halters { - _ = i - <-h.Done.Chan - } - - return p.err -} diff --git a/pjobs_test.go b/pjobs_test.go deleted file mode 100644 index 71192b278..000000000 --- a/pjobs_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "fmt" - "sync/atomic" - "testing" -) - -func Test_ParallelJobs_EarlyShutdown_WaitsForAllGoro(t *testing.T) { - const n = 10000 // total jobs to run - - var errLastOne = fmt.Errorf("the last job has run, and returned this error") - pj := newParallelJobs(100) - nTotal := int64(0) - for i := 0; i < n; i++ { - accepted := pj.run(func(worker int) error { - highpoint := atomic.AddInt64(&nTotal, 1) - switch int(highpoint) { - case n - 1: - return errLastOne - } - return nil - }) - if !accepted { - panic("should have been accepted") - } - } - err := pj.waitForFinish() - tot := atomic.LoadInt64(&nTotal) - if int(tot) != n { - panic(fmt.Sprintf("We didn't run them all? tot=%v, n=%v; pj.jobQ len %v; err='%v'", tot, n, len(pj.jobQ), err)) - } - if err != errLastOne { - panic("expected to see errLastOne") - } - // good: finished cleanly. -} diff --git a/rbf.go b/rbf.go index 64a268281..cee25f35c 100644 --- a/rbf.go +++ b/rbf.go @@ -15,15 +15,12 @@ package pilosa import ( - "bytes" "fmt" "io" - "io/ioutil" "math" "os" "strings" "sync" - "sync/atomic" "github.com/molecula/featurebase/v2/rbf" rbfcfg "github.com/molecula/featurebase/v2/rbf/cfg" @@ -76,8 +73,6 @@ func (w *RbfDBWrapper) CleanupTx(tx Tx) { w.muDb.Lock() delete(w.openTx, r) - //vv("rbf CleanupTx gid %v about to call r.o.dbs.Cleanup(tx.Sn=%v)", curGID(), tx.Sn()) - r.o.dbs.Cleanup(tx) // release the read/write lock. w.muDb.Unlock() } @@ -186,20 +181,12 @@ type RBFTx struct { initialIndex string tx *rbf.Tx o Txo - sn int64 // serial number Db *RbfDBWrapper done bool mu sync.Mutex // protect done as it changes state } -func (tx *RBFTx) IsDone() (done bool) { - tx.mu.Lock() - done = tx.done - tx.mu.Unlock() - return -} - func (tx *RBFTx) DBPath() string { return tx.tx.DBPath() } @@ -210,17 +197,13 @@ func (tx *RBFTx) Type() string { func (tx *RBFTx) Rollback() { tx.tx.Rollback() - - // must happen after actual rollback tx.Db.CleanupTx(tx) } func (tx *RBFTx) Commit() (err error) { err = tx.tx.Commit() - - // must happen after actual commit tx.Db.CleanupTx(tx) - return + return err } func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { @@ -412,10 +395,6 @@ func (tx *RBFTx) Min(index, field, view string, shard uint64) (uint64, bool, err return tx.tx.Min(rbfName(index, field, view, shard)) } -func (tx *RBFTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - return tx.tx.UnionInPlace(rbfName(index, field, view, shard), others...) -} - // CountRange returns the count of hot bits in the start, end range on the fragment. // roaring.countRange counts the number of bits set between [start, end). func (tx *RBFTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { @@ -426,24 +405,8 @@ func (tx *RBFTx) OffsetRange(index, field, view string, shard uint64, offset, st return tx.tx.OffsetRange(rbfName(index, field, view, shard), offset, start, end) } -func (tx *RBFTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {} - -func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { - return tx.tx.ImportRoaringBits(rbfName(index, field, view, shard), rit, clear, log, rowSize, data) -} - -func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") - } - var buf bytes.Buffer - sz, err = rbm.WriteTo(&buf) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") - } - return ioutil.NopCloser(&buf), sz, err +func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { + return tx.tx.ImportRoaringBits(rbfName(index, field, view, shard), rit, clear, log, rowSize) } func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { @@ -452,39 +415,6 @@ func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring return b.Iterator() } -func (tx *RBFTx) Pointer() string { - return fmt.Sprintf("%p", tx) -} - -func (tx *RBFTx) Dump(short bool, shard uint64) { - tx.tx.Dump(short, shard) -} - -// Readonly is true if the transaction is not read-and-write, but only doing reads. -func (tx *RBFTx) Readonly() bool { - return !tx.tx.Writable() -} - -func (tx *RBFTx) Group() *TxGroup { - return tx.o.Group -} - -func (tx *RBFTx) Options() Txo { - return tx.o -} - -func (tx *RBFTx) Sn() int64 { - return tx.sn -} - -func (tx *RBFTx) UseRowCache() bool { - // since RFB returns memory mapped data, we can't use - // the rowCache without first making a copy. - // So we only use the rowCache if the copy is - // enabled. - return storage.EnableRowCache() -} - func (tx *RBFTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { return tx.tx.ApplyFilter(rbfName(index, field, view, shard), ckey, filter) } @@ -591,20 +521,16 @@ func (w *RbfDBWrapper) OpenDB() error { return nil } -var globalNextTxSnRBFTx int64 - func (w *RbfDBWrapper) NewTx(write bool, initialIndex string, o Txo) (_ Tx, err error) { tx, err := w.db.Begin(write) if err != nil { return nil, err } - sn := atomic.AddInt64(&globalNextTxSnRBFTx, 1) rtx := &RBFTx{ tx: tx, initialIndex: initialIndex, o: o, - sn: sn, Db: w, } @@ -629,20 +555,6 @@ func (w *RbfDBWrapper) DeleteFragment(index, field, view string, shard uint64, f return tx.Commit() } -func (w *RbfDBWrapper) DeleteDBPath(dbs *DBShard) error { - path := dbs.pathForType(rbfTxn) - return os.RemoveAll(path) -} - func (w *RbfDBWrapper) OpenListString() (r string) { return "rbf OpenListString not implemented yet" } - -func (w *RbfDBWrapper) OpenSnList() (slc []int64) { - w.muDb.Lock() - for v := range w.openTx { - slc = append(slc, v.sn) - } - w.muDb.Unlock() - return -} diff --git a/rbf/cursor_internal_test.go b/rbf/cursor_internal_test.go index 3d8dca800..e426dda5b 100644 --- a/rbf/cursor_internal_test.go +++ b/rbf/cursor_internal_test.go @@ -59,20 +59,12 @@ func TestCursor_RoaringImport(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. _ = rowSet if changed != 1 { t.Fatalf("expected 1 changed, got %v", changed) } - if false { - cur, err := tx.cursor(name) - PanicOn(err) - - cur.dump() - - _ = cur.tx.dumpAllPages(true) - } } func TestCursor_RoaringImport_clear_bits(t *testing.T) { @@ -93,7 +85,7 @@ func TestCursor_RoaringImport_clear_bits(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) _ = rowSet if changed != 1 { @@ -104,21 +96,12 @@ func TestCursor_RoaringImport_clear_bits(t *testing.T) { clear = true itr2 := getRoaringIter([]uint64{1}...) - changed, rowSet, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil) + changed, rowSet, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize) PanicOn(err) _ = rowSet if changed != 1 { t.Fatalf("expected 1 changed on clear true, got %v", changed) } - - if false { - cur, err := tx.cursor(name) - PanicOn(err) - - cur.dump() - - _ = cur.tx.dumpAllPages(true) - } } func TestCursor_RoaringImport_two_leaves(t *testing.T) { @@ -152,20 +135,12 @@ func TestCursor_RoaringImport_two_leaves(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. _ = rowSet if changed != 6000 { t.Fatalf("expected 6000 bits changed, got %v", changed) } - if false { - cur, err := tx.cursor(name) - PanicOn(err) - - cur.dump() - - _ = cur.tx.dumpAllPages(true) - } } func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) { @@ -211,7 +186,7 @@ func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) { defer tx.Rollback() //vv("DONE WITH Add()") - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != expectedBitsChanged-3000 { t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged-3000, changed) @@ -219,37 +194,23 @@ func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) { //vv("changed on set is %v", changed) //vv("about to do itr2, that starts with key %v", itr2.ContainerKeys()[0]) - changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil) + changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize) PanicOn(err) if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) } - //vv("done with itr2") - - dump := func() { - cur, err := tx.cursor(name) - PanicOn(err) - //cur.dump() - _ = cur.tx.dumpAllPages(true) - } - _ = dump - //dump() - //vv("now clear") - // now clear clear = true //itr3 := getRoaringIter(want[len(want)-3000:]...) itr3 := getRoaringIter(want...) - changed, _, err = tx.ImportRoaringBits(name, itr3, clear, false, rowSize, nil) + changed, _, err = tx.ImportRoaringBits(name, itr3, clear, false, rowSize) PanicOn(err) if changed != expectedBitsChanged { // cursor_internal_test.go:235: expected 2,724,000 bits changed, got 2,721,000 t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed) } - - //dump() } func TestCursor_RoaringImport_auto_many_leaves(t *testing.T) { @@ -292,34 +253,21 @@ func TestCursor_RoaringImport_auto_many_leaves(t *testing.T) { defer tx.Rollback() //vv("DONE WITH Add()") - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != expectedBitsChanged { t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed) } - //vv("changed on set is %v", changed) - - dump := func() { - cur, err := tx.cursor(name) - PanicOn(err) - //cur.dump() - _ = cur.tx.dumpAllPages(true) - } - _ = dump - //dump() - //vv("now clear") // now clear clear = true itr2 := getRoaringIter(want...) - changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil) + changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize) PanicOn(err) if changed != expectedBitsChanged { t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed) } - - //dump() } func TestCursor_putBranchCellsHandlesLotsOfNewBranchesAtTheRoot(t *testing.T) { @@ -430,8 +378,6 @@ func TestCursor_putBranchCellsHandlesLotsOfNewBranchesAtTheRoot(t *testing.T) { err = c.putBranchCells(0, branches) PanicOn(err) - - //c.tx.dumpAllPages(true) } func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) { @@ -475,29 +421,18 @@ func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - dump := func() { - cur, err := tx.cursor(name) - PanicOn(err) - //cur.dump() - _ = cur.tx.dumpAllPages(true) - } - _ = dump for i := 0; i < NbranchCells; i++ { itr := getRoaringIter(want[i*3000 : (i+1)*3000]...) - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) } - //vv("changed on set is %v", changed) - //dump() } - //vv("now clear") - // now clear clear = true @@ -505,13 +440,11 @@ func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) { itr := getRoaringIter(want[i*3000 : (i+1)*3000]...) - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) } - //vv("changed on set is %v", changed) - //dump() } } @@ -558,35 +491,22 @@ func TestCursor_from_B_to_C(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - dump := func() { - cur, err := tx.cursor(name) - PanicOn(err) - //cur.dump() - _ = cur.tx.dumpAllPages(true) - } - _ = dump itr := getRoaringIter(want[:6000]...) - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 6000 { t.Fatalf("expected %v bits changed, got %v", 6000, changed) } - //vv("changed on set is %v", changed) - //dump() //vv("STARTING TO ADD C") itr = getRoaringIter(want[6000:9000]...) - changed, _, err = tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err = tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) } - //vv("changed on set is %v", changed) - //dump() - - //vv("now clear") // now clear clear = true @@ -595,12 +515,10 @@ func TestCursor_from_B_to_C(t *testing.T) { itr := getRoaringIter(want[i*3000 : (i+1)*3000]...) - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) // failing here got 0 } - //vv("changed on clear is %v", changed) - //dump() } } diff --git a/rbf/cursorx.go b/rbf/cursorx.go index 0916d2c36..9b2a61111 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -174,7 +174,7 @@ func intoContainer(l leafCell, tx *Tx, replacing *roaring.Container, target []by orig := l.Data var cpMaybe []byte var mapped bool - if storage.EnableRowCache() || tx.db.cfg.DoAllocZero { + if storage.RowCacheEnabled() || tx.db.cfg.DoAllocZero { // make a copy, otherwise the rowCache will see corrupted data // or mmapped data that may disappear. cpMaybe = target[:len(orig)] @@ -191,7 +191,7 @@ func intoContainer(l leafCell, tx *Tx, replacing *roaring.Container, target []by case ContainerTypeBitmapPtr: _, bm, _ := tx.leafCellBitmap(toPgno(cpMaybe)) cloneMaybe := bm - if storage.EnableRowCache() { + if storage.RowCacheEnabled() { cloneMaybe = (*[1024]uint64)(unsafe.Pointer(&target[0]))[:1024] copy(cloneMaybe, bm) } @@ -217,7 +217,7 @@ func toContainer(l leafCell, tx *Tx) (c *roaring.Container) { orig := l.Data var cpMaybe []byte var mapped bool - if storage.EnableRowCache() || tx.db.cfg.DoAllocZero { + if storage.RowCacheEnabled() || tx.db.cfg.DoAllocZero { // make a copy, otherwise the rowCache will see corrupted data // or mmapped data that may disappear. cpMaybe = make([]byte, len(orig)) @@ -234,7 +234,7 @@ func toContainer(l leafCell, tx *Tx) (c *roaring.Container) { case ContainerTypeBitmapPtr: _, bm, _ := tx.leafCellBitmap(toPgno(cpMaybe)) cloneMaybe := bm - if storage.EnableRowCache() { + if storage.RowCacheEnabled() { cloneMaybe = make([]uint64, len(bm)) copy(cloneMaybe, bm) } diff --git a/rbf/db.go b/rbf/db.go index f1623ded8..4f88de259 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -319,7 +319,7 @@ func (db *DB) Close() (err error) { // least a single hot bit inside the db // in order to return hasAnyRecords true. // -// HasData is used by backend migration and blue/green checks. +// HasData is used by backend migration. // // If there is a disk error we return (false, error), so always // check the error before deciding if hasAnyRecords is valid. diff --git a/rbf/tx.go b/rbf/tx.go index f126cfbf9..9ecbe9a02 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -23,7 +23,6 @@ import ( "sync" "github.com/benbjohnson/immutable" - "github.com/molecula/featurebase/v2/hash" "github.com/molecula/featurebase/v2/roaring" txkey "github.com/molecula/featurebase/v2/short_txkey" . "github.com/molecula/featurebase/v2/vprint" @@ -1299,27 +1298,6 @@ func (tx *Tx) Min(name string) (uint64, bool, error) { return uint64((cell.Key << 16) | uint64(cell.firstValue(tx))), true, nil } -func (tx *Tx) UnionInPlace(name string, others ...*roaring.Bitmap) error { - rbm, err := tx.RoaringBitmap(name) - PanicOn(err) - - rbm.UnionInPlace(others...) - // iterate over the containers that changed within rbm, and write them back to disk. - - it, found := rbm.Containers.Iterator(0) - _ = found // don't care about the value of found, because first containerKey might be > 0 - - for it.Next() { - containerKey, rc := it.Value() - - // TODO: only write the changed ones back, as optimization? - // Compare to ImportRoaringBits. - err := tx.PutContainer(name, containerKey, rc) - PanicOn(err) - } - return nil -} - // roaring.countRange counts the number of bits set between [start, end). func (tx *Tx) CountRange(name string, start, end uint64) (uint64, error) { tx.mu.RLock() @@ -1538,130 +1516,7 @@ func (si *emptyContainerIterator) Value() (uint64, *roaring.Container) { return 0, nil } -func (tx *Tx) Dump(short bool, shard uint64) { - fmt.Println(tx.DumpString(short, shard)) -} -func (tx *Tx) DumpString(short bool, shard uint64) (r string) { - - r = "allkeys:[\n" - - // grab root records, for a list of bitmaps. - records, err := tx.RootRecords() - PanicOn(err) - n := 0 - - for itr := records.Iterator(); !itr.Done(); { - name, _ := itr.Next() - - c, err := tx.cursor(name.(string)) - PanicOn(err) - defer c.Close() - - err = c.First() // First will rewind to beginning. - if err == io.EOF { - r += "" - n++ - continue - } - PanicOn(err) - for { - err := c.Next() - if err == io.EOF { - break - } - PanicOn(err) - - elem := &c.stack.elems[c.stack.top] - leafPage, _, err := c.tx.readPage(elem.pgno) - PanicOn(err) - cell := readLeafCell(leafPage, elem.index) - - ckey := cell.Key - ct := toContainer(cell, tx) - - s := stringOfCkeyCt(ckey, ct, name.(string), short, true) - r += s - n++ - } - } - if n == 0 { - return "" - } - // note that we can have a bitmap present, but it can be empty - r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + "\n" - - return "rbf-" + r -} - -func containerToBytes(ct *roaring.Container) []byte { - - ty := roaring.ContainerType(ct) - switch ty { - case roaring.ContainerNil: - PanicOn("nil container") - case roaring.ContainerArray: - return fromArray16(roaring.AsArray(ct)) - case roaring.ContainerBitmap: - return fromArray64(roaring.AsBitmap(ct)) - case roaring.ContainerRun: - return fromInterval16(roaring.AsRuns(ct)) - } - PanicOn(fmt.Sprintf("unknown container type '%v'", int(ty))) - return nil -} - -func bitmapAsString(rbm *roaring.Bitmap) (r string) { - r = "c(" - slc := rbm.Slice() - width := 0 - s := "" - for _, v := range slc { - if width == 0 { - s = fmt.Sprintf("%v", v) - } else { - s = fmt.Sprintf(", %v", v) - } - width += len(s) - r += s - if width > 70 { - r += ",\n" - width = 0 - } - } - if width == 0 && len(r) > 2 { - r = r[:len(r)-2] - } - return r + ")" -} - -func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string, short, showHash bool) (s string) { - - hsh := "" - if showHash { - by := containerToBytes(ct) - hsh = hash.Blake3sum16(by) - } - - cts := roaring.NewSliceContainers() - cts.Put(ckey, ct) - rbm := &roaring.Bitmap{Containers: cts} - srbm := bitmapAsString(rbm) - - var pre string - if len(rrName) > 0 { - pre = txkey.PrefixToString([]byte(rrName)) - } - bkey := pre + fmt.Sprintf("ckey@%020d", ckey) - - s = fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hsh, ct.N()) - - if !short { - s += " ......." + srbm + "\n" - } - return -} - -func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { +func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { // begin write boilerplate if tx.db == nil { diff --git a/rbf/tx_test.go b/rbf/tx_test.go index 771e17722..f7cd78d6a 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -16,14 +16,12 @@ package rbf_test import ( "fmt" - "math" "math/rand" "sync" "testing" "time" "github.com/molecula/featurebase/v2/rbf" - txkey "github.com/molecula/featurebase/v2/short_txkey" ) func TestTx_CommitRollback(t *testing.T) { @@ -496,29 +494,6 @@ func BenchmarkTx_Contains(b *testing.B) { } } -func TestTx_Dump(t *testing.T) { - db := MustOpenDB(t) - defer MustCloseDB(t, db) - tx := MustBegin(t, db, true) - defer tx.Rollback() - - index, field, view, shard := "i", "f", "v", uint64(15) - - nm := rbfName(index, field, view, shard) - - if err := tx.CreateBitmap(nm); err != nil { - t.Fatal(err) - } else if _, err := tx.Add(nm, 0x00000001, 0x00000002, 0x00010003, 0x00030004); err != nil { - t.Fatal(err) - } - - // test that we don't crash, and get *something* back - s := tx.DumpString(true, math.MaxUint64) - if s == "" { - panic("should have had 3 containers!") - } -} - func TestTx_CreateBitmap(t *testing.T) { t.Run("Bulk", func(t *testing.T) { db := MustOpenDB(t) @@ -542,7 +517,3 @@ func TestTx_CreateBitmap(t *testing.T) { } }) } - -func rbfName(index, field, view string, shard uint64) string { - return string(txkey.Prefix(index, field, view, shard)) -} diff --git a/rbf/util.go b/rbf/util.go index fede5c190..7a44120a7 100644 --- a/rbf/util.go +++ b/rbf/util.go @@ -15,13 +15,16 @@ package rbf import ( "fmt" - "io" "strings" txkey "github.com/molecula/featurebase/v2/short_txkey" . "github.com/molecula/featurebase/v2/vprint" ) +// we don't currently use dumpAllPages but it's tricky enough to get right +// that it's probably worth keeping as a debugging tool. +var _ = (*Tx).dumpAllPages + func (tx *Tx) dumpAllPages(showLeaves bool) error { infos, err := tx.PageInfos() @@ -213,56 +216,6 @@ func prefixToString(s string) (ret string) { return txkey.PrefixToString([]byte(s)) } -func (c *Cursor) dump() { - fmt.Printf("\n Cursor %p has bitmaps:\n%v\n", c, c.debugStringBitmaps()) -} - -var _ = (&Cursor{}).dump -var _ = (&Cursor{}).debugStringBitmaps - -func (c_orig *Cursor) debugStringBitmaps() (r string) { - - // work with a totally new Cursor, so we don't impact our current cursor - // so any test using the cursor isn't disturbed. - c2 := Cursor{tx: c_orig.tx} - c2.stack.elems[0] = c_orig.stack.elems[0] - err := c2.First() - if err != nil { - if err == io.EOF { - // ok, can be empty - return "" - } else { - panic(err) - } - } - n := 0 - for { - err := c2.Next() - if err == io.EOF { - break - } - PanicOn(err) - - //instead of cell := c2.cell() - elem := &c2.stack.elems[c2.stack.top] - leafPage, _, err := c2.tx.readPage(elem.pgno) - PanicOn(err) - cell := readLeafCell(leafPage, elem.index) - - ckey := cell.Key - ct := toContainer(cell, c2.tx) - const short = true - s := stringOfCkeyCt(ckey, ct, "", short, true) - r += s - n++ - } - - if n == 0 { - return "" - } - return -} - ///////////////// happy linter var _ = printMetaPage diff --git a/roaring/roaring.go b/roaring/roaring.go index e35487a3e..4fd8a50c0 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2434,13 +2434,13 @@ func (b *Bitmap) writeOp(op *op) error { // Iterator returns a new iterator for the bitmap. func (b *Bitmap) Iterator() *Iterator { - itr := NewIterator(&BitmapIteratorFinder{b}) + itr := &Iterator{bitmap: b} itr.Seek(0) return itr } func (b *Bitmap) IteratorAt(start uint64) *Iterator { - itr := NewIterator(&BitmapIteratorFinder{b}) + itr := &Iterator{bitmap: b} itr.Seek(start) return itr } @@ -2701,37 +2701,18 @@ type BitmapInfo struct { From, To uintptr // if set, indicates the address range used when unpacking } -type IteratorFinder interface { - FindIterator(uint64) (ContainerIterator, bool) - Close() -} -type BitmapIteratorFinder struct { - bitmap *Bitmap -} - -func (bif *BitmapIteratorFinder) FindIterator(seek uint64) (ContainerIterator, bool) { - return bif.bitmap.Containers.Iterator(seek) -} -func (bif *BitmapIteratorFinder) Close() {} - // Iterator represents an iterator over a Bitmap. type Iterator struct { - finder IteratorFinder + bitmap *Bitmap citer ContainerIterator key uint64 c *Container j, k int32 // i: container; j: array index, bit index, or run index; k: offset within the run } -// NewIterator requires f as an IteratorFinder, it will -// crash if f is nil. -func NewIterator(f IteratorFinder) *Iterator { - return &Iterator{finder: f} -} - -func (itr *Iterator) Close() { - itr.finder.Close() -} +// This exists because we used to support a backend which needed it, and I +// don't want to re-experience the joy of figuring out where close calls are needed. +func (itr *Iterator) Close() {} // Seek moves to the first value equal to or greater than `seek`. func (itr *Iterator) Seek(seek uint64) { @@ -2740,7 +2721,7 @@ func (itr *Iterator) Seek(seek uint64) { itr.k = -1 // Move to the correct container. - itr.citer, _ = itr.finder.FindIterator(highbits(seek)) + itr.citer, _ = itr.bitmap.Containers.Iterator(highbits(seek)) if !itr.citer.Next() { itr.c = nil return // eof @@ -7537,10 +7518,6 @@ func (c *Container) Difference(other *Container) *Container { return difference(c, other) } -func NewSliceContainers() *sliceContainers { - return newSliceContainers() -} - // Slice returns an array of the values in the container as uint16. // Do NOT modify the result; it could be the container's actual storage. func (c *Container) Slice() (r []uint16) { diff --git a/rrtx.go b/rrtx.go index d5504cecb..247ae77f1 100644 --- a/rrtx.go +++ b/rrtx.go @@ -15,9 +15,7 @@ package pilosa import ( - "bytes" "fmt" - "io" "os" "path/filepath" "sort" @@ -49,27 +47,10 @@ type RoaringTx struct { w *RoaringWrapper } -func (tx *RoaringTx) IsDone() (done bool) { - tx.mu.Lock() - done = tx.done - tx.mu.Unlock() - return -} - func (tx *RoaringTx) Type() string { return RoaringTxn } -func (tx *RoaringTx) Dump(short bool, shard uint64) { - o := tx.o - o.Shard = shard - fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys(short, false, o)) -} - -func (tx *RoaringTx) UseRowCache() bool { - return storage.EnableRowCache() -} - // based on view.openFragments() func roaringMapOfShards(optionalViewPath string) (shardMap map[uint64]bool, err error) { @@ -112,10 +93,6 @@ func roaringMapOfShards(optionalViewPath string) (shardMap map[uint64]bool, err return } -func (tx *RoaringTx) Pointer() string { - return fmt.Sprintf("%p", tx) -} - // NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE // the transaction Commits or Rollsback. func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { @@ -127,33 +104,16 @@ func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roa // ImportRoaringBits return values changed and rowSet will be inaccurate if // the data []byte is supplied. This mimics the traditional roaring-per-file // and should be faster. -func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { +func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { f, err := tx.getFragment(index, field, view, shard) if err != nil { return 0, nil, err } - if len(data) > 0 { - // changed and rowSet are ignored anyway when len(data) > 0; - // when we are called from fragment.fillFragmentFromArchive() - // which is the only place the data []byte is supplied. - // blueGreenTx also turns off the checks in this case. - return 0, nil, f.readStorageFromArchive(bytes.NewBuffer(data)) - } changed, rowSet, err = f.storage.ImportRoaringRawIterator(rit, clear, true, rowSize) return } -func (tx *RoaringTx) Readonly() bool { - return !tx.write -} - -func (tx *RoaringTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - frag, err := tx.getFragment(index, field, view, shard) - PanicOn(err) - frag.incrementOpN(changedN) -} - func (c *RoaringTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { return GenericApplyFilter(c, index, field, view, shard, ckey, filter) } @@ -279,15 +239,6 @@ func (tx *RoaringTx) Min(index, field, view string, shard uint64) (uint64, bool, return v, ok, nil } -func (tx *RoaringTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return err - } - b.UnionInPlace(others...) - return nil -} - func (tx *RoaringTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) { b, err := tx.bitmap(index, field, view, shard) if err != nil { @@ -381,33 +332,6 @@ func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.B return frag.storage, nil } -func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - file, err := os.Open(fragmentPathForRoaring) // open the fragment file - if err != nil { - return nil, -1, err - } - fi, err := file.Stat() - if err != nil { - return nil, -1, errors.Wrap(err, "statting") - } - sz = fi.Size() - r = file - return -} - -func (tx *RoaringTx) Group() *TxGroup { - return tx.o.Group -} - -func (tx *RoaringTx) Options() Txo { - return tx.o -} - -// Sn retreives the serial number of the Tx. -func (tx *RoaringTx) Sn() int64 { - return tx.sn -} - func roaringGetFieldView2Shards(idx *Index) (vs *FieldView2Shards, err error) { vs = NewFieldView2Shards() @@ -651,18 +575,12 @@ func (w *RoaringWrapper) CleanupTx(tx Tx) { return } r.done = true - - r.o.dbs.Cleanup(tx) // release the read/write lock. } func (w *RoaringWrapper) OpenListString() (r string) { return "RoaringWrapper.OpenListString() not yet implemented" } -func (w *RoaringWrapper) OpenSnList() (slc []int64) { - return nil -} - func (w *RoaringWrapper) CloseDB() error { return errors.New("CloseDB not supported in roaring") } @@ -721,21 +639,12 @@ func (w *RoaringWrapper) IsClosed() (closed bool) { return } -func (w *RoaringWrapper) DeleteDBPath(dbs *DBShard) (err error) { - //vv("RoaringWrapper.DeleteDBPath called on dbs = '%#v'", dbs) - path := dbs.pathForType(roaringTxn) - return os.RemoveAll(path) -} - func (w *RoaringWrapper) DeleteField(index, field, fieldPath string) error { //vv("RoaringWrapper.DeleteField(index = '%v', field = '%v', fieldPath = '%v'", index, field, fieldPath) // match txn sn count vs lmdb/etc. atomic.AddInt64(&globalNextTxSnRoaring, 1) - // under blue-green bolt_roaring, the directory will not be found, b/c bolt will have - // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: - // "If the path does not exist, RemoveAll returns nil (no error)" err := os.RemoveAll(fieldPath) if err != nil { return errors.Wrap(err, "removing directory") diff --git a/server.go b/server.go index 8912d38bb..63077302b 100644 --- a/server.go +++ b/server.go @@ -334,8 +334,8 @@ func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption { } // OptServerStorageConfig is a functional option on Server used to specify the -// transactional-storage backend to use, resulting in RoaringTx, RbfTx, -// BadgerTx, or a blueGreen* Tx being used for all Tx interface calls. +// transactional-storage backend to use, resulting in RoaringTx or RbfTx +// being used for all Tx interface calls. func OptServerStorageConfig(cfg *storage.Config) ServerOption { return func(s *Server) error { s.holderConfig.StorageConfig = cfg diff --git a/server/cluster_test.go b/server/cluster_test.go index 1f7d59b14..51b8c8ebf 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -142,19 +142,6 @@ func TestClusterResize_EmptyNodes(t *testing.T) { // Ensure that adding a node correctly resizes the cluster. func TestClusterResize_AddNode(t *testing.T) { - // Why are we skipping this test under blue-green with Roaring? - // - // We see red test: during resize during importRoaringBits - // PILOSA_STORAGE_BACKEND=rbf_roaring go test -v -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0" -run TestClusterResize_AddNode/"ContinuousShards" - // green: - // PILOSA_STORAGE_BACKEND=roaring_rbf go test -v -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0" -run TestClusterResize_AddNode/"ContinuousShards" - // - // but rbf_badger and badger_rbf are both green (use the same data values for containers). - // - // Conclude: roaring reads a different size of data []byte in (due to ops log) bits vs others (RBF, badger), so - // we can't do blue-green with roaring on this test. - skipTestUnderBlueGreenWithRoaring(t) - t.Run("NoData", func(t *testing.T) { clus := test.MustRunCluster(t, 3) defer clus.Close() @@ -344,8 +331,6 @@ func TestClusterResize_AddNode(t *testing.T) { // Ensure that adding a node correctly resizes the cluster. func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { - skipTestUnderBlueGreenWithRoaring(t) - t.Run("WithIndex", func(t *testing.T) { c := test.MustRunCluster(t, 3) defer c.Close() @@ -630,12 +615,3 @@ func TestClusterMutualTLS(t *testing.T) { t.Fatal(err) } } - -func skipTestUnderBlueGreenWithRoaring(t *testing.T) { - src := pilosa.CurrentBackend() - if strings.Contains(src, "_") { - if strings.Contains(src, "roaring") { - t.Skip("skip for roaring blue-green") - } - } -} diff --git a/server/config.go b/server/config.go index dc6ae08ae..344440ded 100644 --- a/server/config.go +++ b/server/config.go @@ -207,15 +207,8 @@ type Config struct { // Storage.Backend determines which Tx implementation the holder/Index will // use; one of the available transactional-storage engines. Choices are - // listed in the string constants below. Should be one of "roaring","bolt", - // "rbf", "bolt_roaring", "roaring_bolt", "rbf_roaring", "roaring_rbf", - // "bolt_rbf", "rbf_bolt", or any later addition. The engines with _ - // underscore indicate use of a blueGreenTx with a comparison of values back - // from each Tx method, and a panic if they differ. This is an effective - // test for consistency. If "rbf_roaring" is specified, then the roaring - // values are the ones actually returned from the blueGreenTx. If - // "roaring_rbf" is chosen, then the RBF values are the ones actually - // returned from the blueGreenTx. + // listed in the string constants below. Should be one of "roaring" or + // "rbf". Storage *storage.Config `toml:"storage"` // RowcacheOn, if true, turns on the row cache for all storage backends. diff --git a/stattx.go b/stattx.go index 9dc5ab5f9..65a87fdf3 100644 --- a/stattx.go +++ b/stattx.go @@ -16,7 +16,6 @@ package pilosa import ( "fmt" - "io" "math" "runtime" "sort" @@ -151,8 +150,7 @@ type kall int // constants for kall argument to callStats.add() const ( - kIncrementOpN kall = iota - kNewTxIterator + kNewTxIterator kall = iota kImportRoaringBits kRollback kCommit @@ -169,23 +167,14 @@ const ( kCount kMax kMin - kUnionInPlace kCountRange kOffsetRange - kRoaringBitmapReader - kSliceOfShards kLast // mark the end, always keep this last. The following aren't tracked atm: kType - kDump - kReadonly - kPointer - kUseRowCache ) func (k kall) String() string { switch k { - case kIncrementOpN: - return "kIncrementOpN" case kNewTxIterator: return "kNewTxIterator" case kImportRoaringBits: @@ -220,62 +209,21 @@ func (k kall) String() string { return "kMax" case kMin: return "kMin" - case kUnionInPlace: - return "kUnionInPlace" case kCountRange: return "kCountRange" case kOffsetRange: return "kOffsetRange" - case kRoaringBitmapReader: - return "kRoaringBitmapReader" - case kSliceOfShards: - return "kSliceOfShards" case kLast: return "kLast" case kType: return "kType" - case kDump: - return "kDump" - case kReadonly: - return "kReadonly" - case kPointer: - return "kPointer" - case kUseRowCache: - return "kUseRowCache" } PanicOn(fmt.Sprintf("unknown kall '%v'", int(k))) return "" } -var _ = newStatTx // happy linter -var _ = kPointer -var _ = kUseRowCache -var _ = kType -var _ = kDump -var _ = kReadonly - var _ Tx = (*statTx)(nil) -func (c *statTx) Group() *TxGroup { - return c.b.Group() -} - -func (c *statTx) Options() Txo { - return c.b.Options() -} - -//IncrementOpN -func (c *statTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - me := kIncrementOpN - - t0 := time.Now() - defer func() { - c.stats.add(me, time.Since(t0)) - }() - - c.b.IncrementOpN(index, field, view, shard, changedN) -} - func (c *statTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { me := kNewTxIterator @@ -286,7 +234,7 @@ func (c *statTx) NewTxIterator(index, field, view string, shard uint64) *roaring return c.b.NewTxIterator(index, field, view, shard) } -func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { +func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { me := kImportRoaringBits t0 := time.Now() @@ -299,25 +247,7 @@ func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit PanicOn(r) } }() - return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) -} - -func (c *statTx) Dump(short bool, shard uint64) { - c.b.Dump(short, shard) -} - -func (c *statTx) Readonly() bool { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Readonly() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.Readonly() -} - -func (tx *statTx) Pointer() string { - return fmt.Sprintf("%p", tx) + return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize) } func (c *statTx) Rollback() { @@ -421,14 +351,6 @@ func (c *statTx) RemoveContainer(index, field, view string, shard uint64, key ui return c.b.RemoveContainer(index, field, view, shard, key) } -func (c *statTx) UseRowCache() bool { - return c.b.UseRowCache() -} - -func (c *statTx) IsDone() (done bool) { - return c.b.IsDone() -} - func (c *statTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { me := kAdd @@ -586,23 +508,6 @@ func (c *statTx) Min(index, field, view string, shard uint64) (uint64, bool, err return c.b.Min(index, field, view, shard) } -func (c *statTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - me := kUnionInPlace - - t0 := time.Now() - defer func() { - c.stats.add(me, time.Since(t0)) - }() - - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.UnionInPlace(index, field, view, shard, others...) -} - func (c *statTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { me := kCountRange @@ -636,32 +541,10 @@ func (c *statTx) OffsetRange(index, field, view string, shard, offset, start, en return c.b.OffsetRange(index, field, view, shard, offset, start, end) } -func (c *statTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - me := kRoaringBitmapReader - - t0 := time.Now() - defer func() { - c.stats.add(me, time.Since(t0)) - }() - - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) -} - func (c *statTx) Type() string { return c.b.Type() } -// Sn retreives the serial number of the Tx. -func (c *statTx) Sn() int64 { - return c.b.Sn() -} - func (c *statTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) { return c.b.GetSortedFieldViewList(idx, shard) } diff --git a/storage/cache.go b/storage/cache.go index 7fda5af83..fe65a90f3 100644 --- a/storage/cache.go +++ b/storage/cache.go @@ -31,6 +31,6 @@ func SetRowCacheOn(on bool) { } } -func EnableRowCache() bool { +func RowCacheEnabled() bool { return atomic.LoadInt64(&enableRowcache) == 1 } diff --git a/tx.go b/tx.go index 13fd9fded..767969671 100644 --- a/tx.go +++ b/tx.go @@ -15,8 +15,6 @@ package pilosa import ( - "io" - "github.com/molecula/featurebase/v2/roaring" txkey "github.com/molecula/featurebase/v2/short_txkey" //txkey "github.com/molecula/featurebase/v2/txkey" @@ -45,8 +43,8 @@ const writable = true // that have not been committed. type Tx interface { - // Type returns "roaring", "rbf", "bolt", "badger_roaring", or one of the other - // blue-green Tx types at the top of txfactory.go + // Type returns "roaring", "rbf", or one of the other + // Tx types at the top of txfactory.go Type() string // Rollback must be called the end of read-only transactions. Either @@ -65,32 +63,6 @@ type Tx interface { // Commit makes the updates in the Tx visible to subsequent transactions. Commit() error - // IsDone must return true if Rollback() or Commit() has already - // been called. Otherwise it must return false. This allows - // DBWrapper.CleanupTx(tx Tx) to be idempotent. - IsDone() bool - - // Readonly returns the flag this transaction was created with - // during NewTx. If the transaction is writable, it will return false. - Readonly() bool - - // UseRowCache is used by fragment.go unprotectedRow() to determine - // dynamically at runtime if RoaringTx - // are in use, which for continuity wants to continue to use the - // rowCache, or if other storage engines (RBF, Badger) are in - // use, which will mean that the bitmap data stored by the - // rowCache can disappear as it is un-mmap-ed, causing crashes. - UseRowCache() bool - - // IncrementOpN updates internal statistics with the changedN provided. - IncrementOpN(index, field, view string, shard uint64, changedN int) - - // Pointer gives us a memory address for the underlying - // transaction for debugging. - // It is public because we use it in roaring to report invalid - // container memory access outside of a transaction. - Pointer() string - // NewTxIterator returns it, a *roaring.Iterator whose it.Next() will // successively return each uint64 stored in the conceptual roaring.Bitmap // for the specified fragment. @@ -103,8 +75,7 @@ type Tx interface { // Return value 'found' is true when the ckey container was present. // ckey of 0 gives all containers (in the fragment). // - // ContainerIterator must not have side-effects. blueGreenTx will - // call it at the very beginning of commit to verify db contents. + // ContainerIterator must not have side-effects. // // citer.Close() must be called when the client is done using it. ContainerIterator(index, field, view string, shard uint64, ckey uint64) (citer roaring.ContainerIterator, found bool, err error) @@ -154,9 +125,6 @@ type Tx interface { // Min Min(index, field, view string, shard uint64) (uint64, bool, error) - // UnionInPlace - UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error - // CountRange CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) @@ -170,31 +138,9 @@ type Tx interface { // If clear is true, the bits from rit are cleared, otherwise they are set in the // specifed fragment. // - // The data argument can be nil, its ignored for RBF/BadgerTx. It is supplied to - // RoaringTx.ImportRoaringBits() in fragment.go fragment.fillFragmentFromArchive() - // to do the traditional fragment.readStorageFromArchive() which - // does some in memory field/view/fragment metadata updates. - // It makes blueGreenTx testing viable too. - // // ImportRoaringBits return values changed and rowSet may be inaccurate if // the data []byte is supplied (the RoaringTx implementation neglects this for speed). - ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) - - RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) - - // Group returns nil or the TxGroup that this Tx is a part of. - Group() *TxGroup - - // Dump is for debugging, what does this Tx see as its database? - Dump(short bool, shard uint64) - - // Options returns the options used to create this Tx. This - // can be implementd by embedding Txo, and Txo provides the - // Options() method. - Options() Txo - - // Sn retreives the serial number of the Tx. - Sn() int64 + ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) // GetSortedFieldViewList gets the set of FieldView(s) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) @@ -202,51 +148,6 @@ type Tx interface { GetFieldSizeBytes(index, field string) (uint64, error) } -// Closer is used by Finders -type Closer interface { - Close() -} - -type Dumper interface { - // Dump is for debugging, what does this Tx see as its database? - AllDump() -} - -// TxStore has operations that will create and commit multiple -// Tx on a backing store. -type TxStore interface { - - // DeleteFragment deletes all the containers in a fragment. - // - // This is not in a Tx because it will often do too many deletes for a single - // transaction, and clients would be suprised to find their Tx had already - // been commited and they are getting an error on double-Commit. - // Instead each TxStore implementation creates and commits as many - // transactions as needed. - // - // Argument frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil. - // If not nil, it must be of type *fragment. If frag is supplied, then - // index must be equal to frag.index, field equal to frag.field, view equal - // to frag.view, and shard equal to frag.shard. - // - DeleteFragment(index, field, view string, shard uint64, frag interface{}) error - - DeleteField(index, field string) error - - // Close shuts down the database. - Close() error -} - -// RawRoaringData used by ImportRoaringBits. -// must be consumable by roaring.newRoaringIterator() -type RawRoaringData struct { - data []byte -} - -func (rr *RawRoaringData) Iterator() (roaring.RoaringIterator, error) { - return roaring.NewRoaringIterator(rr.data) -} - // GenericApplyFilter implements ApplyFilter in terms of tx.ContainerIterator, // as a convenience if a Tx backend hasn't implemented this new function yet. func GenericApplyFilter(tx Tx, index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { diff --git a/txfactory.go b/txfactory.go index 7c0d06add..e28a6fb03 100644 --- a/txfactory.go +++ b/txfactory.go @@ -16,32 +16,22 @@ package pilosa import ( "fmt" - "io" "os" "path" "path/filepath" - "runtime" "strconv" "strings" "sync" - "syscall" - "text/tabwriter" - "github.com/molecula/featurebase/v2/hash" - "github.com/molecula/featurebase/v2/roaring" - txkey "github.com/molecula/featurebase/v2/short_txkey" - "github.com/molecula/featurebase/v2/storage" "github.com/molecula/featurebase/v2/testhook" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck "github.com/pkg/errors" - "github.com/zeebo/blake3" ) // public strings that pilosa/server/config.go can reference const ( RoaringTxn string = "roaring" RBFTxn string = "rbf" - BoltTxn string = "bolt" ) // DetectMemAccessPastTx true helps us catch places in api and executor @@ -267,7 +257,7 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) { // qcx.write reflects the top executor determination // if a write will be done at the end, so we upgrade // the "local" read Tx to be writes, so that they - // don't deadlock against themselves under blue-green. + // don't deadlock against themselves. o.Write = o.Write || qcx.write // In general, we make ALL write transactions local, and never reuse them @@ -305,9 +295,8 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) { if already { return } - o.Group = qcx.Grp tx = qcx.Txf.NewTx(o) - qcx.Grp.AddTx(tx) + qcx.Grp.AddTx(tx, o) return } @@ -351,7 +340,6 @@ func (qcx *Qcx) StartAtomicWriteTx(o Txo) { // new Tx needed tx := qcx.Txf.NewTx(o) qcx.RequiredForAtomicWriteTx = &tx - o := tx.Options() qcx.RequiredTxo = &o return } @@ -374,55 +362,23 @@ func (qcx *Qcx) StartAtomicWriteTx(o Txo) { } } -func (qcx *Qcx) SetRequiredForAtomicWriteTx(tx Tx) { - if tx == nil || NilInside(tx) { - PanicOn("cannot set nil tx in SetRequiredForAtomicWriteTx") - } - qcx.mu.Lock() - qcx.RequiredForAtomicWriteTx = &tx - o := tx.Options() - qcx.RequiredTxo = &o - qcx.mu.Unlock() -} - -func (qcx *Qcx) ClearRequiredForAtomicWriteTx() { - qcx.mu.Lock() - qcx.RequiredForAtomicWriteTx = nil - qcx.RequiredTxo = nil - qcx.mu.Unlock() -} - func (qcx *Qcx) ListOpenTx() string { return qcx.Grp.String() } // TxFactory abstracts the creation of Tx interface-level -// transactions so that RBF, BoltDB, or Roaring-fragment-files, or several +// transactions so that RBF, or Roaring-fragment-files, or several // of these at once in parallel, is used as the storage and transction layer. type TxFactory struct { typeOfTx string - mu sync.Mutex - - types []txtype // blue-green split individually here + typ txtype dbsClosed bool // idemopotent CloseDB() dbPerShard *DBPerShard holder *Holder - - blueGreenReg *blueGreenRegistry - - // allow holder to activate blue-green checking only - // once we have synced both sides at start up time. - blueGreenOff bool - - isBlueGreen bool -} - -func (f *TxFactory) Types() []txtype { - return f.types } // integer types for fast switch{} @@ -432,7 +388,6 @@ const ( noneTxn txtype = 0 roaringTxn txtype = 1 // these don't really have any transactions rbfTxn txtype = 2 - boltTxn txtype = 4 ) // DirectoryName just returns a string version of the transaction type. We @@ -445,73 +400,41 @@ func (ty txtype) DirectoryName() string { return "roaring" case rbfTxn: return "rbf" - case boltTxn: - return "boltdb" } PanicOn(fmt.Sprintf("unkown txtype %v", int(ty))) return "" } func (txf *TxFactory) NeedsSnapshot() (b bool) { - for _, ty := range txf.types { - switch ty { - case roaringTxn: - b = true - return - } - } - return + return txf.typ == roaringTxn } -func MustBackendToTxtype(backend string) (types []txtype) { - var srcs []string +func MustBackendToTxtype(backend string) (typ txtype) { if strings.Contains(backend, "_") { - srcs = strings.Split(backend, "_") - if len(srcs) != 2 { - PanicOn("only two blue-green comparisons permitted") - } - } else { - srcs = append(srcs, backend) + panic("blue-green comparisons removed") } - for i, s := range srcs { - switch s { - case RoaringTxn: // "roaring" - types = append(types, roaringTxn) - case RBFTxn: // "rbf" - types = append(types, rbfTxn) - case BoltTxn: // "bolt" - types = append(types, boltTxn) - default: - PanicOn(fmt.Sprintf("unknown backend '%v'", s)) - } - if i == 1 { - if types[1] == types[0] { - PanicOn(fmt.Sprintf("cannot blue-green the same backend on both arms: '%v'", s)) - } - } + switch backend { + case RoaringTxn: // "roaring" + return roaringTxn + case RBFTxn: // "rbf" + return rbfTxn } - return + panic(fmt.Sprintf("unknown backend '%v'", backend)) } // NewTxFactory always opens an existing database. If you // want to a fresh database, os.RemoveAll on dir/name ahead of time. // We always store files in a subdir of holderDir. func NewTxFactory(backend string, holderDir string, holder *Holder) (f *TxFactory, err error) { - types := MustBackendToTxtype(backend) + typ := MustBackendToTxtype(backend) f = &TxFactory{ - types: types, + typ: typ, typeOfTx: backend, holder: holder, } - if len(types) == 2 { - f.blueGreenReg = newBlueGreenReg(types) - f.isBlueGreen = true - // blue-green can never use the rowCache. - storage.SetRowCacheOn(false) - } - f.dbPerShard = f.NewDBPerShard(types, holderDir, holder) + f.dbPerShard = f.NewDBPerShard(typ, holderDir, holder) if f.hasRBF() { holder.Logger.Infof("rbf config = %#v", holder.cfg.RBFConfig) @@ -526,15 +449,6 @@ func (f *TxFactory) Open() error { return f.dbPerShard.LoadExistingDBs() } -// UseRowCache can be more "global" than Tx at the moment, because -// we are sharing the same bool flag in rbf at the moment. If -// this changes then fragment.openStorage() will need a new way -// to determine if it should use the rowCache. Currently it -// doesn't have a tx Tx parameter, so we use the Txf instead. -func (f *TxFactory) UseRowCache() bool { - return storage.EnableRowCache() -} - // Txo holds the transaction options type Txo struct { Write bool @@ -544,23 +458,14 @@ type Txo struct { Shard uint64 dbs *DBShard - per *DBPerShard - - Group *TxGroup - - blueGreenOff bool -} - -func (o Txo) String() string { - return fmt.Sprintf("Txo{Write:%v, Index:%v Shard:%v Group:%p}", o.Write, o.Index.name, o.Shard, o.Group) } func (f *TxFactory) TxType() string { return f.typeOfTx } -func (f *TxFactory) TxTypes() []txtype { - return f.types +func (f *TxFactory) TxTyp() txtype { + return f.typ } func (f *TxFactory) DeleteIndex(name string) (err error) { @@ -577,10 +482,6 @@ func (f *TxFactory) DeleteFragmentFromStore( return f.dbPerShard.DeleteFragment(index, field, view, shard, frag) } -func (f *TxFactory) DumpAll() { - f.dbPerShard.DumpAll() -} - // IndexUsageDetails computes the sum of filesizes used by the node, broken down // by index, field, fragments and keys. func (f *TxFactory) IndexUsageDetails(isClosing func() bool) (map[string]IndexUsage, uint64, error) { @@ -769,7 +670,6 @@ func directoryUsage(fname string, recursive bool) (uint64, error) { // CloseIndex is a no-op. This seems to be in place for debugging purposes. func (f *TxFactory) CloseIndex(idx *Index) error { - //idx.Dump("CloseIndex") return nil } @@ -790,24 +690,24 @@ func init() { } } -// TxGroup holds a set of read and a set of write transactions -// that will en-mass have Rollback() (for the read set) and -// Commit() (for the write set) called on +// TxGroup holds a set of read transactions +// that will en-mass have Rollback() (for the read set) called on // them when TxGroup.Finish() is invoked. // Alternatively, TxGroup.Abort() will call Rollback() // on all Tx group memebers. +// +// It used to have writes but we never actually used that because +// of the Qcx needing to make every commit get its own transaction. type TxGroup struct { mu sync.Mutex fac *TxFactory reads []Tx - writes []Tx finished bool all map[grpkey]Tx } type grpkey struct { - write bool index string shard uint64 } @@ -822,7 +722,7 @@ func (g *TxGroup) AlreadyHaveTx(o Txo) (tx Tx, already bool) { mustHaveIndexShard(&o) g.mu.Lock() defer g.mu.Unlock() - key := grpkey{write: o.Write, index: o.Index.name, shard: o.Shard} + key := grpkey{index: o.Index.name, shard: o.Shard} tx, already = g.all[key] return } @@ -830,21 +730,14 @@ func (g *TxGroup) AlreadyHaveTx(o Txo) (tx Tx, already bool) { func (g *TxGroup) String() (r string) { g.mu.Lock() defer g.mu.Unlock() - if len(g.reads) == 0 && len(g.writes) == 0 { + if len(g.reads) == 0 { return "" } - - i := 0 r += "\n" - for _, tx := range g.reads { - r += fmt.Sprintf("[%v]read: _sn_ %v %v, \n", i, tx.Sn(), tx.Options()) - i++ + for i, tx := range g.reads { + r += fmt.Sprintf("[%v]read: %#v,\n", i, tx) } - for _, tx := range g.writes { - r += fmt.Sprintf("[%v]write: _sn_ %v %v, \n", i, tx.Sn(), tx.Options()) - i++ - } - return + return r } // NewTxGroup @@ -857,7 +750,7 @@ func (f *TxFactory) NewTxGroup() (g *TxGroup) { } // AddTx adds tx to the group. -func (g *TxGroup) AddTx(tx Tx) { +func (g *TxGroup) AddTx(tx Tx, o Txo) { g.mu.Lock() defer g.mu.Unlock() if g.finished { @@ -867,15 +760,9 @@ func (g *TxGroup) AddTx(tx Tx) { PanicOn("Cannot add nil Tx to TxGroup") } - if tx.Readonly() { - g.reads = append(g.reads, tx) - } else { - g.writes = append(g.writes, tx) - } - o := tx.Options() - mustHaveIndexShard(&o) + g.reads = append(g.reads, tx) - key := grpkey{write: o.Write, index: o.Index.name, shard: o.Shard} + key := grpkey{index: o.Index.name, shard: o.Shard} prior, ok := g.all[key] if ok { PanicOn(fmt.Sprintf("already have Tx in group for this, we should have re-used it! prior is '%v'; tx='%v'", prior, tx)) @@ -893,15 +780,6 @@ func (g *TxGroup) FinishGroup() (err error) { PanicOn("in TxGroup.Finish(): TxGroup already finished") } g.finished = true - for i, tx := range g.writes { - _ = i - err0 := tx.Commit() - if err0 != nil { - if err == nil { - err = err0 // keep the first error, but Commit them all. - } - } - } for _, r := range g.reads { r.Rollback() } @@ -923,9 +801,6 @@ func (g *TxGroup) AbortGroup() { for _, r := range g.reads { r.Rollback() } - for _, tx := range g.writes { - tx.Rollback() - } } func (f *TxFactory) NewTx(o Txo) (txn Tx) { @@ -935,12 +810,6 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) { } }() - if f.isBlueGreen { - f.mu.Lock() - o.blueGreenOff = f.blueGreenOff - f.mu.Unlock() - } - indexName := "" if o.Index != nil { indexName = o.Index.name @@ -963,10 +832,8 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) { if dbs.Shard != o.Shard { PanicOn(fmt.Sprintf("asked for o.Shard=%v but got dbs.Shard=%v", int(o.Shard), int(dbs.Shard))) } - //vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.types='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.types, dbs.W) - - o.dbs = dbs // our specific database per shard. - o.per = f.dbPerShard // for top level debug Dumps + //vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.typ='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.typ, dbs.W) + o.dbs = dbs tx, err := dbs.NewTx(o.Write, indexName, o) if err != nil { @@ -984,8 +851,6 @@ func (ty txtype) String() string { return "roaring" case rbfTxn: return "rbf" - case boltTxn: - return "bolt" } PanicOn(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty))) return "" @@ -1022,147 +887,6 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64, return } -// hashOnly means only show the value hash, not the content bits. -// showOps means display the ops log. -func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool, o Txo) (r string) { - paths, err := listFilesUnderDir(idx.path, false, "", true) - PanicOn(err) - index := idx.name - - r = "allkeys:[\n" - n := 0 - for _, relpath := range paths { - field, view, shard, err := fragmentSpecFromRoaringPath(relpath) - if err != nil { - continue // ignore .meta paths - } - if shard != o.Shard { - continue // only print the shard the Txo is on. - } - abspath := idx.path + sep + relpath - - s, _, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard, showOps, hashOnly, os.Stdout) - PanicOn(err) - //r += fmt.Sprintf("path:'%v' fragment contains:\n") + s - //if s == "" { - //s = "" - //} - r += s - n++ - } - if n == 0 { - return "" - } - // note that we can have a bitmap present, but it can be empty - r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + "\n" - - return "roaring-" + r -} - -func RoaringFragmentChecksum(path string, index, field, view string, shard uint64) (r string, hotbits int) { - defer func() { - r := recover() - if r != nil { - PanicOn(fmt.Sprintf("caught PanicOn on path='%v', index='%v', field='%v', view='%v', shard='%v': %v", - path, index, field, view, shard, r)) - } - }() - hasher := blake3.New() - showOps := false - hashOnly := true - hash, hotbits, err := stringifiedRawRoaringFragment(path, index, field, view, shard, showOps, hashOnly, hasher) - PanicOn(err) - fmt.Fprintf(hasher, "%v/%v/%v/%v/%v", index, field, view, shard, hash) - var buf [16]byte - _, _ = hasher.Digest().Read(buf[0:]) - return fmt.Sprintf("%x", buf), hotbits - -} - -func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64, showOps, hashOnly bool, w io.Writer) (r string, hotbits int, err error) { - - var info roaring.BitmapInfo - _ = info - var f *os.File - f, err = os.Open(path) - PanicOn(err) - if err != nil { - return - } - - var fi os.FileInfo - fi, err = f.Stat() - PanicOn(err) - if err != nil { - return - } - // Memory map the file. - data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) - if err != nil { - err = errors.Wrap(err, "mmapping") - return - } - defer func() { - err := syscall.Munmap(data) - if err != nil { - PanicOn(fmt.Errorf("loadRawRoaringContainer: munmap failed: %v", err)) - } - PanicOn(f.Close()) - }() - - // Attach the mmap file to the bitmap. - var rbm *roaring.Bitmap - rbm, _, err = roaring.InspectBinary(data, true, &info) - if err != nil { - err = errors.Wrap(err, "inspecting") - return - } - - //cmd.DisplayInfo(info) - // inlined - if showOps { - pC := pointerContext{ - from: info.From, - to: info.To, - } - if info.ContainerCount > 0 { - printContainers(w, info, pC) - } - if info.Ops > 0 { - printOps(w, info) - } - } - - citer, found := rbm.Containers.Iterator(0) - _ = found // probably gonna use just the Ops log instead, so don't PanicOn if !found. - - for citer.Next() { - ckey, ct := citer.Value() - by := containerToBytes(ct) - hash := hash.Blake3sum16(by) - - cts := roaring.NewSliceContainers() - cts.Put(ckey, ct) - rbm := &roaring.Bitmap{Containers: cts} - - var srbm string - if !hashOnly { - srbm = BitmapAsString(rbm) - } - - bkey := txkey.ToString(txkey.Key(index, field, view, shard, ckey)) - - n := ct.N() - hotbits += int(n) - r += fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, n) - if !hashOnly { - r += " ......." + srbm + "\n" - } - } - - return -} - // listFilesUnderDir returns the paths of files found under directory root. // If includeRoot is true, it returns the full path, otherwise paths are relative to root. // If requriedSuffix is supplied, the returned file paths will end in that, @@ -1218,130 +942,6 @@ func fileSize(name string) (int64, error) { return fi.Size(), nil } -var _ = fileSize // happy linter - -func containerToBytes(ct *roaring.Container) []byte { - ty := roaring.ContainerType(ct) - switch ty { - case roaring.ContainerNil: - PanicOn("nil roaring.Container") - case roaring.ContainerArray: - return fromArray16(roaring.AsArray(ct)) - case roaring.ContainerBitmap: - return fromArray64(roaring.AsBitmap(ct)) - case roaring.ContainerRun: - return fromInterval16(roaring.AsRuns(ct)) - } - PanicOn(fmt.Sprintf("unknown roaring.Container type '%v'", int(ty))) - return nil -} - -type pointerContext struct { - from, to uintptr -} - -func printOps(w io.Writer, info roaring.BitmapInfo) { - fmt.Fprintln(w, " Ops:") - tw := tabwriter.NewWriter(w, 0, 8, 0, '\t', 0) - fmt.Fprintf(tw, " \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE") - printed := 0 - for _, op := range info.OpDetails { - fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size) - printed++ - } - tw.Flush() -} - -func (p *pointerContext) pretty(c roaring.ContainerInfo) string { - var pointer string - if c.Mapped { - if c.Pointer >= p.from && c.Pointer < p.to { - pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from) - } else { - pointer = fmt.Sprintf("!0x%x!", c.Pointer) - } - } else { - pointer = fmt.Sprintf("0x%x", c.Pointer) - } - return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer) -} - -// stolen from ctl/inspect.go -func printContainers(w io.Writer, info roaring.BitmapInfo, pC pointerContext) { - fmt.Fprintln(w, " Containers:") - tw := tabwriter.NewWriter(w, 0, 8, 0, '\t', 0) - fmt.Fprintf(tw, " \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n") - fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS") - c1s := info.Containers - c2s := info.OpContainers - l1 := len(c1s) - l2 := len(c2s) - i1 := 0 - i2 := 0 - var c1, c2 roaring.ContainerInfo - c1.Key = ^uint64(0) - c2.Key = ^uint64(0) - c1e := false - c2e := false - if i1 < l1 { - c1 = c1s[i1] - i1++ - c1e = true - } - if i2 < l2 { - c2 = c2s[i2] - i2++ - c2e = true - } - printed := 0 - for c1e || c2e { - c1used := false - c2used := false - var key uint64 - c1fmt := "-\t\t\t" - c2fmt := "-\t\t\t" - // If c2 exists, we'll always prefer its flags, - // if it doesn't, this gets overwritten. - flags := c2.Flags - if !c2e || (c1e && c1.Key < c2.Key) { - c1fmt = pC.pretty(c1) - key = c1.Key - c1used = true - flags = c1.Flags - } else if !c1e || (c2e && c2.Key < c1.Key) { - c2fmt = pC.pretty(c2) - key = c2.Key - c2used = true - } else { - // c1e and c2e both set, and neither key is < the other. - c1fmt = pC.pretty(c1) - c2fmt = pC.pretty(c2) - key = c1.Key - c1used = true - c2used = true - } - if c1used { - if i1 < l1 { - c1 = c1s[i1] - i1++ - } else { - c1e = false - } - } - if c2used { - if i2 < l2 { - c2 = c2s[i2] - i2++ - } else { - c2e = false - } - } - fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags) - printed++ - } - tw.Flush() -} - var _ = anyGlobalDBWrappersStillOpen // happy linter func anyGlobalDBWrappersStillOpen() bool { @@ -1351,225 +951,19 @@ func anyGlobalDBWrappersStillOpen() bool { if globalRbfDBReg.Size() != 0 { return true } - if globalBoltReg.Size() != 0 { - return true - } return false } -func (f *TxFactory) blueGreenOnIfRunningBlueGreen() { - if len(f.types) == 2 { - f.blueGreenOff = false - } -} - -func (f *TxFactory) blueGreenOffIfRunningBlueGreen() { - if len(f.types) == 2 { - f.blueGreenOff = true - } -} - func (f *TxFactory) hasRoaring() bool { - return f.types[0] == roaringTxn || (len(f.types) > 1 && f.types[1] == roaringTxn) + return f.typ == roaringTxn } func (f *TxFactory) hasRBF() bool { - return f.types[0] == rbfTxn || (len(f.types) > 1 && f.types[1] == rbfTxn) + return f.typ == rbfTxn } var _ = (&TxFactory{}).hasRoaring // happy linter -func (f *TxFactory) blueHasData() (hasData bool, err error) { - if len(f.types) != 2 { - return false, nil - } - return f.dbPerShard.HasData(0) -} - -func (f *TxFactory) greenHasData() (hasData bool, err error) { - n := len(f.types) - switch n { - case 1: - return f.dbPerShard.HasData(0) - case 2: - return f.dbPerShard.HasData(1) - } - err = fmt.Errorf("unsupported len(f.types): %v; must be 1 or 2", n) - PanicOn(err) - return -} - -// green2blue is called at the very end of Holder.Open(), so -// we know that the holder is ready to go, knowing its holder.Indexes(), fields, -// view, shards, and other metadata if any. -// -// Called by test Test_TxFactory_UpdateBlueFromGreen_OnStartup() in -// txfactory_internal_test.go as well. -// -// This is a noop if we aren't running under a blue_green PILOSA_STORAGE_BACKEND. -func (f *TxFactory) green2blue(holder *Holder) (err0 error) { - - // Holder.Open will always call us, even without blue_green. Which is fine. - // We are just a no-op in that case. - if len(f.types) != 2 { - return nil - } - - holder.Logger.Infof("green2blue analysis begins.") - - blueDest := f.types[0] - greenSrc := f.types[1] - - if blueDest == roaringTxn { - return fmt.Errorf("error: cannot migrate to 'roaring': not implemented") - } - - idxs := holder.Indexes() - - verifyInsteadOfCopy := false - - blueHasData, err := f.blueHasData() - if err != nil { - return errors.Wrap(err, "TxFactory.green2blue f.blueHasData()") - } - - greenHasData, err := f.greenHasData() - if err != nil { - return errors.Wrap(err, "TxFactory.green2blue f.greenHasData()") - } - if !blueHasData && !greenHasData { - holder.Logger.Infof("no data in blue or green. No migration or verification to do") - return nil - } - // INVAR: blue has data. - if !greenHasData { - holder.Logger.Errorf("cannot migrate from green '%v' because it has no data in it", greenSrc) - return fmt.Errorf("error: cannot migrate from green '%v' because it has no data in it", greenSrc) - } - - nGoro := runtime.NumCPU() - if nGoro < 5 { - // try to get some overlapped IO - nGoro = 5 - } - pj := newParallelJobs(nGoro) - - action := "verify" - if blueHasData { - verifyInsteadOfCopy = true - defer holder.Logger.Infof("bitmap-backend verification done : %v compared to %v", blueDest, greenSrc) - } else { - action = "migrate" - holder.Logger.Infof("bitmap-backend migration starting: populating %v from %v with %v threads", blueDest, greenSrc, nGoro) - defer holder.Logger.Infof("bitmap-backend migration done : populated %v from %v", blueDest, greenSrc) - } - firstPjobStarted := false - -indexloop: - for k, idx := range idxs { - - // scan directories - blueShards, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(blueDest, idx, "", false) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v') error fetching blueShards", idx.name)) - } - - // scan directories - greenShards, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(greenSrc, idx, "", true) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v') error fetching greenShards", idx.name)) - } - - if verifyInsteadOfCopy { - diff := f.shardSetDiff(blueShards, greenShards) - if diff != "" { - return fmt.Errorf("verifyInsteadOfCopy true, blue[%v]=%#v and green[%v]=%#v have different shards for index '%v': '%v'; stack=\n%v", blueDest, blueShards, greenSrc, greenShards, idx.name, diff, Stack()) - } - - // can also check against meta data - shards := idx.AvailableShards(localOnly).Slice() - meta := make(map[uint64]bool) - for _, shard := range shards { - meta[shard] = true - } - diff2 := f.shardSetDiff(greenShards, meta) - if diff2 != "" { - return fmt.Errorf("green[%v] = '%#v' and meta data '%#v' have different shards for index '%v': %v", greenSrc, greenShards, shards, idx.name, diff2) - } - } - - shardNum := 0 - for shard := range greenShards { - shardNum++ - shnum := shardNum - idx := idx - shard := shard - k := k - fun := func(worker int) error { - - dbs, err := f.dbPerShard.GetDBShard(idx.name, shard, idx) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v', shard='%v')", idx.name, int(shard))) - } - - holder.Logger.Infof("%v progress on index '%v' (%v of %v): on shard '%v' (%v of %v) [worker %v]", - action, idx.name, k+1, len(idxs), shard, shnum, len(greenShards), worker) - - if verifyInsteadOfCopy { - // verify all containers - err = dbs.verifyBlueEqualsGreen() - if err != nil { - return errors.Wrap(err, - fmt.Sprintf("dbs.verifyBlueEqualsGreen(blue='%v', "+ - "green='%v') for index='%v', shard='%v'", - blueDest, greenSrc, idx.name, int(shard))) - } - } else { - // the main copy work - err = dbs.populateBlueFromGreen() - if err != nil { - return errors.Wrap(err, - fmt.Sprintf("dbs.copyGreenToBlue(blue='%v', "+ - "green='%v') for index='%v', shard='%v'", - blueDest, greenSrc, idx.name, int(shard))) - } - } - return nil - } // end of fun definition - - if !pj.run(fun) { - break indexloop - } - if !firstPjobStarted { - firstPjobStarted = true - defer func() { - err1 := pj.waitForFinish() - if err0 == nil { - err0 = err1 - } - }() - } - } - } - return nil -} - -func (f *TxFactory) shardSetDiff(blueShards, greenShards map[uint64]bool) (diff string) { - nb := len(blueShards) - ng := len(greenShards) - if nb != ng { - diff = fmt.Sprintf("blueShard[%v] count = %v; greenShard[%v] count = %v; ", f.types[0], nb, f.types[1], ng) - } - bmg := mapDiff(blueShards, greenShards) // get blue - green - gmb := mapDiff(greenShards, blueShards) // get green - blue - - if len(bmg) == 0 && len(gmb) == 0 { - return "" - } - diff += fmt.Sprintf("shard diff: blueMinusGreen shards: '%#v'; greenMinusBlue shards: '%#v'", bmg, gmb) - return -} - func (f *TxFactory) GetDBShardPath(index string, shard uint64, idx *Index, ty txtype, write bool) (shardPath string, err error) { dbs, err := f.dbPerShard.GetDBShard(index, shard, idx) if err != nil { diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go index b310dbdfe..9cfa9badb 100644 --- a/txfactory_internal_test.go +++ b/txfactory_internal_test.go @@ -15,393 +15,14 @@ package pilosa import ( - "context" - "fmt" - "os" "testing" - "time" - - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck ) -func Test_TxFactory_Qcx_query_context(t *testing.T) { - src := CurrentBackend() - if src == "rbf" || src == "bolt" { - // ok - } else { - t.Skip("this test only for rbf and bolt") - } - - shard := uint64(0) - f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, shard, "") - defer f.Clean(t) - tx.Rollback() - - barrier := NewBarrier() - defer barrier.Close() - - done := make(chan bool) - - setter := func(k int) { - for i := 0; ; i++ { - barrier.WaitAtGate(0) - select { - case <-done: - return - default: - } - // add to the group txn on the txf. - qcx := idx.holder.txf.NewQcx() - - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: idx, Shard: f.shard}) - PanicOn(err) - - // Set bits on the fragment. - if _, err := f.setBit(tx, 120, 1); err != nil { - panic(err) - } else if _, err := f.setBit(tx, 120, 6); err != nil { - panic(err) - } else if _, err := f.setBit(tx, 121, 0); err != nil { - panic(err) - } - // should have two containers set in the fragment. - - // Verify counts on rows. - if n := f.mustRow(tx, 120).Count(); n != 2 { - panic(fmt.Sprintf("unexpected count: %d", n)) - } else if n := f.mustRow(tx, 121).Count(); n != 1 { - panic(fmt.Sprintf("unexpected count: %d", n)) - } - finisher(nil) // hit the write tx.Commit path - // commit the change, and verify it is still there - PanicOn(qcx.Finish()) - qcx.Reset() - - tx, finread, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - PanicOn(err) - if n := f.mustRow(tx, 120).Count(); n != 2 { - panic(fmt.Sprintf("unexpected count (reopen): %d", n)) - } else if n := f.mustRow(tx, 121).Count(); n != 1 { - panic(fmt.Sprintf("unexpected count (reopen): %d", n)) - } - finread(nil) // no-op on reads that are in a group, so must qcx.Abort() to stop them. - qcx.Abort() - qcx.Reset() - } - } - N := 1000 - for i := 0; i < N; i++ { - go setter(i) - } - time.Sleep(time.Second * 1) - close(done) - - // allow all goro to finish before Closing the lmdb.env, otherwise - // we will crash as the goroutines making Tx will try to use the env - // after it is closed. It can take quite a while. - // one writer might be blocking the other... so ask for only N-2 at first - // to avoid deadlock. - barrier.BlockUntil(N - 2) - barrier.UnblockReaders() - time.Sleep(1 * time.Second) -} - -// test TxFactory.green2blue -// -// blue_green starting with an empty or full blue database -// should copy all of green (if blue is empty); or if blue is ull, -// verify that blue has all the same bits as green. -// -// Benefits: a) we start with known identical state so our testing/comparisons can be valid; -// and b) we have an easy migration mechanism, to go from one storage format to another. -// -func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) { - checked := []string{"roaring", "rbf"} - - expectError := false - for _, blue := range checked { - for _, green := range checked { - if blue == green { - continue - } - if blue == "roaring" { - // not supported - expectError = true - } else { - expectError = false - } - blue_green := blue + "_" + green - //vv("setting blue_green to '%v'", blue_green) - - // ============================= - // Begin setup. - // - // Setup happens with green only. - - h, path, err := makeHolder(t, green) - if err != nil { - t.Fatalf("creating holder: %v", err) - } - defer os.RemoveAll(path) - //vv("path = %v", path) - - // we will manually h.Close() below - - // Write bits to separate indexes. - testSetBit(t, h, "i0", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 12345678) - - testOp := testHolderOperator{} - ctx := context.Background() - err = h.Process(ctx, &testOp) - if err != nil { - t.Fatalf("processing holder: %v", err) - } - expected := testHolderOperator{ - indexSeen: 2, indexProcessed: 2, - fieldSeen: 2, fieldProcessed: 2, - viewSeen: 2, viewProcessed: 2, - fragmentSeen: 3, fragmentProcessed: 3, - } - if testOp != expected { - t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp) - } - - // verify data is there - rowID := uint64(100) - colID := uint64(200) - _, _ = rowID, colID - testMustHaveBit(t, h, "i0", "f", rowID, colID) - testMustHaveBit(t, h, "i1", "f", 100, 200) - testMustHaveBit(t, h, "i1", "f", 100, 12345678) - - //vv("about to reopen; blue_green = '%v' but PILOSA_STORAGE_BACKEND='%v'", blue_green, os.Getenv("PILOSA_STORAGE_BACKEND")) - //h.DumpAllShards() - - //vv("after dump, about to close") - h.Close() - - //vv("after close, about to re-open") - - // can we re.Open the same holder h? hopefully without a problem. - PanicOn(h.Open()) - - //vv("h.Open() re-open worked; blue_green = '%v'; dump; with PILOSA_STORAGE_BACKEND='%v'", blue_green, os.Getenv("PILOSA_STORAGE_BACKEND")) - //h.DumpAllShards() - - testMustHaveBit(t, h, "i0", "f", rowID, colID) // panic here, colID 200 bit was cold. - testMustHaveBit(t, h, "i1", "f", 100, 200) - testMustHaveBit(t, h, "i1", "f", 100, 12345678) - h.Close() - - //vv("successful re-open and then Close again of h.") - - // check that we can open a NewHolder on green, on same path, and still see our bits. - // Because the NewHolder is the code that creates and configures TxFactory as blue_green. - cfg := mustHolderConfig() - cfg.StorageConfig.Backend = green - h2 := NewHolder(path, cfg) - PanicOn(h2.Open()) - - testMustHaveBit(t, h2, "i0", "f", rowID, colID) - testMustHaveBit(t, h2, "i1", "f", 100, 200) - testMustHaveBit(t, h2, "i1", "f", 100, 12345678) - h2.Close() - - // verify that blue does not have it. - // open a new holder on path, just looking at blue. - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue - h3 := NewHolder(path, cfg) - PanicOn(h3.Open()) - - testMustNotHaveBit(t, h3, "i0", "f", rowID, colID) - testMustNotHaveBit(t, h3, "i1", "f", 100, 200) - testMustNotHaveBit(t, h3, "i1", "f", 100, 12345678) - - h3.Close() - - // ============================= - // Setup done. On to actual test. - - // Opening in blue_green mode means that once Holder.Open() - // returns without error, the blue and green databases are - // identical. - // Since blue is empty, the blue database will get synched up - // with the green during Holder.Open(). - - // open a holder with path again, now looking at both blue and green. - // The Holder.Open should do the migration from green, populating blue. - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue_green - h4 := NewHolder(path, cfg) - - //vv("about to h4.Open we should populate blue from green") - err = h4.Open() - if expectError { - if err == nil { - panic("expected error since migration to roaring not supported") - } - } else { - PanicOn(err) - } - - testMustHaveBit(t, h4, "i0", "f", rowID, colID) - testMustHaveBit(t, h4, "i1", "f", 100, 200) - testMustHaveBit(t, h4, "i1", "f", 100, 12345678) - - //vv("successfully verified populatingBlueFromGreen with blue_green = '%v'", blue_green) - h4.Close() - os.RemoveAll(path) - } - } -} - -// test the situation where we startup blue_green with existing data and -// go to verify it but blue has more data than green. -// That will also cause query divergence. -func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) { - checked := []string{"roaring", "bolt", "rbf"} - - for _, blue := range checked { - for _, green := range checked { - if blue == green { - continue - } - if blue == "roaring" { - // not supported - continue - } - blue_green := blue + "_" + green - - // ============================= - // Begin setup. - // - // Setup happens with green only. - - h, path, err := makeHolder(t, green) - if err != nil { - t.Fatalf("creating holder: %v", err) - } - defer os.RemoveAll(path) - - //vv("on green, which is '%v'", green) - // we will manually h.Close() below - - // Write bits to separate indexes. - testSetBit(t, h, "i0", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 12345678) - - testOp := testHolderOperator{} - ctx := context.Background() - err = h.Process(ctx, &testOp) - if err != nil { - t.Fatalf("processing holder: %v", err) - } - expected := testHolderOperator{ - indexSeen: 2, indexProcessed: 2, - fieldSeen: 2, fieldProcessed: 2, - viewSeen: 2, viewProcessed: 2, - fragmentSeen: 3, fragmentProcessed: 3, - } - if testOp != expected { - t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp) - } - - // verify data is there - rowID := uint64(100) - colID := uint64(200) - _, _ = rowID, colID - testMustHaveBit(t, h, "i0", "f", rowID, colID) - testMustHaveBit(t, h, "i1", "f", 100, 200) - testMustHaveBit(t, h, "i1", "f", 100, 12345678) - - h.Close() - - // verify that blue does not have it. - // open a new holder on path, just looking at blue. - - //vv("on blue, which is '%v'", blue) - - cfg := mustHolderConfig() - cfg.StorageConfig.Backend = blue - h3 := NewHolder(path, cfg) - PanicOn(h3.Open()) - - testMustNotHaveBit(t, h3, "i0", "f", rowID, colID) - testMustNotHaveBit(t, h3, "i1", "f", 100, 200) - testMustNotHaveBit(t, h3, "i1", "f", 100, 12345678) - - h3.Close() - - // ============================= - // Setup done. On to actual test. - - // Opening in blue_green mode means that once Holder.Open() - // returns without error, the blue and green databases are - // identical. - // Since blue is empty, the blue database will get synched up - // with the green during Holder.Open(). - - //vv("on blue_green, which is '%v'", blue_green) - - // open a holder with path again, now looking at both blue and green. - // The Holder.Open should do the migration from green, populating blue. - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue_green - h4 := NewHolder(path, cfg) - PanicOn(h4.Open()) - - testMustHaveBit(t, h4, "i0", "f", rowID, colID) - testMustHaveBit(t, h4, "i1", "f", 100, 200) - testMustHaveBit(t, h4, "i1", "f", 100, 12345678) - h4.Close() - - // now open just blue, and add a bit to a new index, i2. - //vv("on blue, which is '%v'", blue) - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue - h5 := NewHolder(path, cfg) - PanicOn(h5.Open()) - testSetBit(t, h5, "i2", "f", 500, 777) - - //vv("after adding a bit to blue, we have:") - //h5.DumpAllShards() - - h5.Close() - - // now open blue_green. should get a verification failure - // due to the extra bit in blue. - - // BEGIN verficiation that should ERROR out b/c blue has more data. - - // open a holder with path again, now looking at both blue and green. - // The Holder.Open should verify blue against green and notice the extra bit. - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue_green - h6 := NewHolder(path, cfg) - err = h6.Open() - //h6.DumpAllShards() - - if err == nil { - h6.Close() - t.Fatalf("should have had blue-green verification fail on Holder.Open") - } - - h6.Close() - } - } -} - func Test_TxFactory_verifyStringConstantsMatch(t *testing.T) { // txtype.String() method MUST return strings that match - // our const definitions at the top of txfactory.go, or - // else blue-green transactions cannot determine when - // the second transaction is being released in dbshard.go. - check := []txtype{roaringTxn, rbfTxn, boltTxn} - expect := []string{RoaringTxn, RBFTxn, BoltTxn} + // our const definitions at the top of txfactory.go. + check := []txtype{roaringTxn, rbfTxn} + expect := []string{RoaringTxn, RBFTxn} for i, chk := range check { obs := chk.String() if obs != expect[i] { diff --git a/util.go b/util.go index ebbff6e75..6d4b8e09e 100644 --- a/util.go +++ b/util.go @@ -17,19 +17,12 @@ package pilosa // util.go: a place for generic, reusable utilities. import ( - "fmt" - "io/ioutil" "os" - "path/filepath" "reflect" - "sort" - "strings" "syscall" "time" - "unsafe" "github.com/molecula/featurebase/v2/roaring" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck "github.com/pkg/errors" ) @@ -63,229 +56,6 @@ func NilInside(iface interface{}) bool { func highbits(v uint64) uint64 { return v >> 16 } func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) } -func toArray16(a []byte) []uint16 { - return (*[4096]uint16)(unsafe.Pointer(&a[0]))[: len(a)/2 : len(a)/2] -} -func toArray64(a []byte) []uint64 { - return (*[1024]uint64)(unsafe.Pointer(&a[0]))[:1024:1024] -} -func toInterval16(a []byte) []roaring.Interval16 { - return (*[2048]roaring.Interval16)(unsafe.Pointer(&a[0]))[: len(a)/4 : len(a)/4] -} - -func sliceToMap(slc []uint64) (m map[uint64]bool) { - m = make(map[uint64]bool) - for _, v := range slc { - m[v] = true - } - return -} - -// return A - B -func mapDiff(mapA, mapB map[uint64]bool) (r []int) { - for a := range mapA { - _, ok := mapB[a] - if !ok { - r = append(r, int(a)) - } - } - sort.Ints(r) - return -} - -func asInts(a []uint64) (r []int) { - r = make([]int, len(a)) - for i, v := range a { - r[i] = int(v) - } - return -} - -func containerAsString(ckey uint64, rc *roaring.Container) (r string) { - rbm := roaring.NewBitmap() - rbm.Containers.Put(ckey, rc) - return BitmapAsString(rbm) -} - -var _ = containerAsString // happy linter - -func roaringBitmapDiff(a, b *roaring.Bitmap) error { - nA := a.Count() - nB := b.Count() - - slcA := a.Slice() - slcB := b.Slice() - - mapA := sliceToMap(slcA) - mapB := sliceToMap(slcB) - - AminusB := mapDiff(mapA, mapB) - BminusA := mapDiff(mapB, mapA) - - sort.Ints(AminusB) - sort.Ints(BminusA) - - res := fmt.Sprintf("nA = %v; nB = %v;\n", nA, nB) - ndiff := 0 - if nA != nB { - ndiff++ - } - - if len(AminusB) > 0 { - res += fmt.Sprintf("==> AminusB = (len %v) '%#v'; ", len(AminusB), AminusB) - ndiff++ - } - if len(BminusA) > 0 { - res += fmt.Sprintf("\n==> BminusA = (len %v) '%#v'; ", len(BminusA), BminusA) - ndiff++ - } - if ndiff == 0 { - return nil - } - res += fmt.Sprintf("\n ==> A = '%#v'\n ==> B = '%#v'", asInts(slcA), asInts(slcB)) - return errors.New(res) -} - -func dirAsString(path string) (r string) { - r = fmt.Sprintf("dump of directory '%v':\n", path) - files, err := ioutil.ReadDir(path) - PanicOn(err) - for _, f := range files { - r += f.Name() + "\n" - } - return r -} - -var _ = dirAsString // happy linter - -var _ = zeroKeyContainerAsString // happy linter - -// for debugging -func zeroKeyContainerAsString(ct *roaring.Container) (r string) { - cts := roaring.NewSliceContainers() - cts.Put(0, ct) - rbm := &roaring.Bitmap{Containers: cts} - r = fmt.Sprintf("[%v]:", containerTypeNames[roaring.ContainerType(ct)]) + BitmapAsString(rbm) - return -} - -var containerTypeNames = map[byte]string{ - roaring.ContainerArray: "array", - roaring.ContainerBitmap: "bitmap", - roaring.ContainerRun: "run", -} - -func BitmapAsString(rbm *roaring.Bitmap) (r string) { - r = "c(" - slc := rbm.Slice() - width := 0 - s := "" - for _, v := range slc { - if width == 0 { - s = fmt.Sprintf("%v", v) - } else { - s = fmt.Sprintf(", %v", v) - } - width += len(s) - r += s - if width > 70 { - r += ",\n" - width = 0 - } - } - if width == 0 && len(r) > 2 { - r = r[:len(r)-2] - } - return r + ")" -} - -// fromArray16 converts to an 8KB page -func fromArray16(a []uint16) []byte { - if len(a) == 0 { - return []byte{} - } - if len(a) > 4096 { - PanicOn(fmt.Sprintf("cannot put more than 4096 integers into an array container: %v too big", len(a))) - } - return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2] -} - -// fromArray64 converts to an 8KB page -func fromArray64(a []uint64) []byte { - if len(a) == 0 { - return []byte{} - } - return (*[8192]byte)(unsafe.Pointer(&a[0]))[:8192:8192] -} - -// fromInterval16 converts to 8KB page -func fromInterval16(a []roaring.Interval16) []byte { - if len(a) == 0 { - return []byte{} - } - if len(a) > 2048 { - PanicOn(fmt.Sprintf("cannot put more than 2048 roaring.Interval16 into a container: %v too big", len(a))) - } - return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4] -} - -// DiskUse reports the total bytes uses by all files under root -// that match requiredSuffix. requiredSuffix can be empty string. -// Space used by directories is not counted. -func DiskUse(root string, requiredSuffix string) (tot int, err error) { - if !DirExists(root) { - return -1, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root) - } - - err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if info == nil { - PanicOn(fmt.Sprintf("info was nil for path = '%v'", path)) - } - if info.IsDir() { - // skip the size of directories themselves, only summing files. - } else { - sz := info.Size() - if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) { - tot += int(sz) - } - } - return nil - }) - return -} - -// rootDir must exist. Return the size in bytes of the largest sub-directory -// that has the required suffix. The largestSize is from DiskUse() called -// on the sub-dir. DiskUse only counts file size, nothing for directory inodes. -func SubdirLargestDirWithSuffix(rootDir, requiredDirSuffix string) (exists bool, largestSize int, err error) { - if !DirExists(rootDir) { - return false, -1, fmt.Errorf("SubdirExistsWithSuffix error: root directory '%v' not found", rootDir) - } - - err = filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { - if info == nil { - PanicOn(fmt.Sprintf("info was nil for path = '%v'", path)) - } - - if info.IsDir() && strings.HasSuffix(path, requiredDirSuffix) { - exists = true - size, err := DiskUse(path, "") - if err != nil { - // disk error? report it - return err - } - if size > largestSize { - largestSize = size - } - } - return nil - }) - if err != nil { - return exists, -1, err - } - return -} - // called by Holder.hasRoaringData() func roaringFragmentHasData(path string, index, field, view string, shard uint64) (hasData bool, err error) { diff --git a/utils_internal_test.go b/utils_internal_test.go index 76c3aa66f..fa9b666a8 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -15,47 +15,17 @@ package pilosa import ( - "bytes" "fmt" "testing" "time" pnet "github.com/molecula/featurebase/v2/net" - "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/testhook" "github.com/molecula/featurebase/v2/topology" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck ) // utilities used by tests -// mustAddR is a helper for calling roaring.Container.Add() in tests to -// keep the linter happy that we are checking the error. -func mustAddR(changed bool, err error) { - PanicOn(err) -} - -// mustRemove is a helper for calling Tx.Remove() in tests to -// keep the linter happy that we are checking the error. -func mustRemove(changeCount int, err error) { - PanicOn(err) -} - -func getTestBitmapAsRawRoaring(bitsToSet ...uint64) []byte { - b := roaring.NewBitmap() - changed := b.DirectAddN(bitsToSet...) - n := len(bitsToSet) - if changed != n { - panic(fmt.Sprintf("changed=%v but bitsToSet len = %v", changed, n)) - } - buf := bytes.NewBuffer(make([]byte, 0, 100000)) - _, err := b.WriteTo(buf) - if err != nil { - panic(err) - } - return buf.Bytes() -} - // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. func NewTestCluster(tb testing.TB, n int) *cluster { path, err := testhook.TempDir(tb, "pilosa-cluster-") diff --git a/view.go b/view.go index ac94f64d8..911a2d19c 100644 --- a/view.go +++ b/view.go @@ -158,26 +158,28 @@ func (v *view) openWithShardSet(ss *shardSet) error { if nGoro < 4 { nGoro = 4 } - pj := newParallelJobs(nGoro) + var eg errgroup.Group + throttle := make(chan struct{}, nGoro) + for i := range frags { // create a new variable frag on each time through // the loop (instead of i, frag := range frags) // so that the closure run on the // goroutine has its own variable. frag := frags[i] - accepted := pj.run(func(worker int) error { + throttle <- struct{}{} + eg.Go(func() error { + defer func() { + <-throttle + }() if err := frag.Open(); err != nil { return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err) } return nil }) - if !accepted { - // have error/shutting down the pj, so stop - break - } } - err := pj.waitForFinish() + err := eg.Wait() if err != nil { return err }