From dc8702ea3e8df040184dd919c74d6995b3dad4c7 Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 13 Oct 2021 12:38:54 -0500 Subject: [PATCH 1/3] use testhook auditor to track Qcx open/close This also requires doing something to keep the TxGroup in each Qcx from holding its Tx references after the Qcx closes, because otherwise the list of Qcxs that we keep to verify that they all got closed ends up keeping every shared/read-only Tx open forever, resulting in many gigabytes of memory usage when running with the race detector. To avoid having to reason about whether anything would ever access a nil TxGroup, or run through iteratively zeroing maps, we just make a new empty group at that point. --- txfactory.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/txfactory.go b/txfactory.go index 93e9449df..7c0d06add 100644 --- a/txfactory.go +++ b/txfactory.go @@ -31,6 +31,7 @@ import ( "github.com/molecula/featurebase/v2/roaring" txkey "github.com/molecula/featurebase/v2/short_txkey" "github.com/molecula/featurebase/v2/storage" + "github.com/molecula/featurebase/v2/testhook" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck "github.com/pkg/errors" "github.com/zeebo/blake3" @@ -147,6 +148,11 @@ func (q *Qcx) Finish() (err error) { } } err2 := q.Grp.FinishGroup() + // drop the old group so we aren't holding references to all those Tx + q.Grp = q.Txf.NewTxGroup() + if !q.done { + _ = testhook.Closed(q.Txf.holder.Auditor, q, nil) + } q.done = true if err != nil { @@ -164,7 +170,11 @@ func (q *Qcx) Abort() { (*q.RequiredForAtomicWriteTx).Rollback() } q.Grp.AbortGroup() - + // drop the old group so we aren't holding references to all those Tx + q.Grp = q.Txf.NewTxGroup() + if !q.done { + _ = testhook.Closed(q.Txf.holder.Auditor, q, nil) + } q.done = true } @@ -197,6 +207,7 @@ func (f *TxFactory) NewQcx() (qcx *Qcx) { if f.typeOfTx == "roaring" { qcx.isRoaring = true } + _ = testhook.Opened(f.holder.Auditor, qcx, nil) return } From ecd0ecc6d1b8fcef14a642aca6cbee5998e7f22d Mon Sep 17 00:00:00 2001 From: Seebs Date: Wed, 13 Oct 2021 12:39:06 -0500 Subject: [PATCH 2/3] add featurebase to .gitignore we ignored pilosa binaries but we've renamed so now the binary is named featurebase. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 21dbedb4d..2082bd5c0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ release-pilosa-fsck.*.*.tar.gz /log.* /tourna.log.* pilosa +/featurebase *.dot .idea/ .*.swp From ad30a926f4014e3685a68e95cf678ab85256383b Mon Sep 17 00:00:00 2001 From: Seebs Date: Mon, 4 Oct 2021 15:09:51 -0500 Subject: [PATCH 3/3] Giant Commit: drop a bunch of stuff we don't use. These commits are hard to disentagle, and doing them separately means re-modifying the same chunks of code several times before removing it, and similar things. Basically: (1) Drop the bolt backend storage. (2) Drop the blue-green wrapper that compares two backends. (3) Drop unused or barely-used Tx API components from all the remaining backends. (4) Minor related cleanup to simplify things related to these. The boltdb backend existed only to verify RBF. The blue-green wrapper was mostly used to verify RBF, but in practice we had to do a lot of working around that, and it introduced a lot of special cases. Types removed: IteratorFinder: Used only to implement the roaring iterator on top of boltdb, and to complicate the way it worked in roaring. Reverted the complications. Also unexport NewSliceContainers which is used only for that outside of roaring's internals. PortMapper from cluster_internal_test.go: Used only for a test we removed early this year. Never used for anything else. RawRoaringData: Totally unused. TxStore: Totally unused. Functions removed from Tx API, and sometimes corresponding members were removed from structs: * Dump: debugging code, I don't think I found any actually reachable paths to it. * Group: only used for debugging TxGroup stuff * IncrementOpN: only used by fragment, fragment can increment its own opN. * Options: unused? * Pointer: debugging only * Readonly: used only to decide how to handle Tx in a TxGrp, but we never add a non-readonly Tx to a TxGrp. Removed also all the corresponding write-aware stuff. * RoaringBitmapReader: Used exactly once, can just be a bm.WriteTo. * Sn (and OpenSnList): Unused * UnionInPlace: unused and conceptually-invalid; it didn't write to storage and shouldn't have, and was just "create a bitmap then call union-in-place", which we can do directly. * UseRowCache: just checked storage.UseRowCache. Other things removed: The SetRequiredForAtomicWriteTx and ClearRequiredForAtomicWriteTx functions go away, since nothing now seems to be using them? Same for holder_internal_test's `testHasBit` and `testMustNotHaveBit`, which were unused. The DBPerShard "DeleteDBPath" and "HasData" functions and related parts were mostly unused; took out the parts that were never actually being reached. Changed the API of one function to simplify special cases and remove things: * ImportRoaringBits had a special "data" argument which gave it subtly different semantics for RBF and roaring (for roaring, it could produce a roaring bitmap *with ops log*), didn't seem to be adding much. Removed corresponding "readStorageFromArchive" which is not otherwise used. Also took out various debugging/dumping functions that were unused and may have bitrotted. Dropped a test from txfactory_internal_test, and the "pjobs" code, because those two were the only things that needed Barrier and thus idem, which lets us drop two more dependencies. We already have errgroup for grouping things which want to terminate as soon as one of them errors, approximately. To do better we'd have to have context-threading, really. Unbroke the WriteFragment test for non-roaring tests and made it not roaring-only. --- api.go | 12 +- barrier.go | 219 ----- barrier_test.go | 90 -- bluegreentx.go | 1028 ---------------------- bluegreentx_test.go | 100 --- bolt.go | 1612 ----------------------------------- bolt_test.go | 1270 --------------------------- catcher.go | 71 +- cluster_internal_test.go | 50 -- cmd/roaring-migrate/main.go | 2 +- ctl/server.go | 2 +- dbshard.go | 638 ++------------ dbshard_internal_test.go | 22 +- delete_test.go | 1 - executor_test.go | 1 - fragment.go | 91 +- fragment_internal_test.go | 43 +- go.mod | 2 - go.sum | 4 - holder.go | 19 - holder_internal_test.go | 68 -- index.go | 10 - pjobs.go | 115 --- pjobs_test.go | 51 -- rbf.go | 94 +- rbf/cursor_internal_test.go | 110 +-- rbf/cursorx.go | 8 +- rbf/db.go | 2 +- rbf/tx.go | 147 +--- rbf/tx_test.go | 29 - rbf/util.go | 55 +- roaring/roaring.go | 37 +- rrtx.go | 93 +- server.go | 4 +- server/cluster_test.go | 24 - server/config.go | 11 +- stattx.go | 123 +-- storage/cache.go | 2 +- tx.go | 107 +-- txfactory.go | 676 +-------------- txfactory_internal_test.go | 385 +-------- util.go | 230 ----- utils_internal_test.go | 30 - view.go | 16 +- 44 files changed, 223 insertions(+), 7481 deletions(-) delete mode 100644 barrier.go delete mode 100644 barrier_test.go delete mode 100644 bluegreentx.go delete mode 100644 bluegreentx_test.go delete mode 100644 bolt.go delete mode 100644 bolt_test.go delete mode 100644 pjobs.go delete mode 100644 pjobs_test.go diff --git a/api.go b/api.go index ca3506a0e..7bcc0dda7 100644 --- a/api.go +++ b/api.go @@ -1510,9 +1510,7 @@ func addClearToImportOptions(opts []ImportOption) []ImportOption { return append(opts, OptImportOptionsClear(true)) } -// Import avoids re-writing a bajillion tests to be transaction-aware by allowing a nil pQcx. -// It is convenient for some tests, particularly those in loops, to pass a nil qcx and -// treat the Import as having been commited when we return without error. We make it so. +// Import does the top-level importing. func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts ...ImportOption) (err error) { if req.Clear { opts = addClearToImportOptions(opts) @@ -1648,8 +1646,8 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest, return errors.Wrap(err, "committing") } -// ImportValue avoids re-writing a bajillion tests by allowing a nil pQcx. -// Then we will commit before returning. +// ImportValue is a wrapper around the common code in ImportValueWithTx, which +// currently just translates req.Clear into a clear ImportOption. func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error { if req.Clear { opts = addClearToImportOptions(opts) @@ -2654,9 +2652,7 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64 if err != nil { return err } - //need to find the path to the db - //will not work on blue green - db := dbs.W[0] + db := dbs.W finalPath := db.Path() + "/data" tempPath := finalPath + ".tmp" o, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) diff --git a/barrier.go b/barrier.go deleted file mode 100644 index b150dc486..000000000 --- a/barrier.go +++ /dev/null @@ -1,219 +0,0 @@ -// home https://github.com/glycerine/lmdb-go -// Copyright (c) 2020, the lmdb-go authors -// Copyright (c) 2015, Bryan Matsuo -// All rights reserved. - -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: - -// Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. - -// Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. - -// Neither the name of the author nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. - -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package pilosa - -import ( - "github.com/glycerine/idem" -) - -// Barrier allows us to temporarily halt all readers, so that -// a writer can commit alone and thus compact the db. -// The Barrier starts unblocked, alllowing passage to any -// caller of WaitAtGate(). -type Barrier struct { - wait chan *appointment // send upon entering the waiting room. - halt *idem.Halter - blockReqCh chan *blockReq - unblockCh chan *unblock -} - -type blockReq struct { - count int - done chan struct{} -} - -func newBlockReq(count int) *blockReq { - return &blockReq{ - count: count, - done: make(chan struct{}), - } -} - -type appointment struct { - id int - done chan struct{} -} - -func newAppointment(id int) *appointment { - return &appointment{ - id: id, - done: make(chan struct{}), - } -} - -// NewBarrier is either open, allowing immediate passage, -// or blocked, halting all callers at WaitAtGate() -// until the barrier is opened. By default it is open. -// -// Barrier.Close() must be called when the barrier -// is no longer needed to avoid a goroutine leak. -func NewBarrier() (b *Barrier) { - b = &Barrier{ - wait: make(chan *appointment), // waiters indicate they are waiting for the gate by sending here. - halt: idem.NewHalter(), - blockReqCh: make(chan *blockReq), - unblockCh: make(chan *unblock), - } - go func() { - defer b.halt.Done.Close() - - var waitlist []*appointment - var curBlockReq *blockReq - - for { - select { - case br := <-b.blockReqCh: - if br.count == 0 { - close(br.done) - continue - } - if curBlockReq == nil { - // good, changing state from open to closed barrier. - } else { - panic("got 2nd block request atop of first") - } - curBlockReq = br - //vv("barrier: request to block for %v waiters", br.count) - if len(waitlist) != 0 { - panic("had waiters when we were open, internal/client bug") - } - case appt := <-b.wait: - //vv("barrier.wait sees appt = '%#v' and curBlockReq = '%#v'", appt, curBlockReq) - if curBlockReq == nil { - close(appt.done) - continue - } - waitlist = append(waitlist, appt) - n := len(waitlist) - th := curBlockReq.count - if th < 0 { - // infinite waiters. we block everybody until we - // see an unblock request. - continue - } - if n >= th { - close(curBlockReq.done) - curBlockReq = nil - } - case ub := <-b.unblockCh: - for _, appt := range waitlist { - close(appt.done) - } - waitlist = nil - curBlockReq = nil - close(ub.done) - case <-b.halt.ReqStop.Chan: - return - } - } - }() - return -} - -// WaitAtGate will return immediately -// if the barrier is unblocked. Otherwise -// it will not return until another -// goroutine unblocks the barrier. -func (b *Barrier) WaitAtGate(id int) { - appt := newAppointment(id) - select { - case b.wait <- appt: - select { - case <-appt.done: - case <-b.halt.ReqStop.Chan: - } - case <-b.halt.ReqStop.Chan: - } -} - -// Close should be called to stop the -// barrier's background goroutine when -// you are done using the barrier. -func (b *Barrier) Close() { - b.halt.ReqStop.Close() - <-b.halt.Done.Chan -} - -type unblock struct { - done chan struct{} -} - -func newUnblock() *unblock { - return &unblock{ - done: make(chan struct{}), - } -} - -// Unblock lets all waiting goroutines resume execution. -func (b *Barrier) UnblockReaders() { - ub := newUnblock() - select { - case b.unblockCh <- ub: - select { - case <-ub.done: - case <-b.halt.ReqStop.Chan: - } - case <-b.halt.ReqStop.Chan: - } -} - -// BlockUntil is called with a count, the -// number of waiters required to be present and waiting -// at the gate before call returns. -// A count of < 0 will return immediately and raise -// the barrier to any number of arriving readers. -// A count of 0 is a no-op. -// -// Otherwise we raise the barrier -// and wait until we have seen count other goroutines waiting -// on it. -// -// We return without releasing the waiters. Call -// Open when you want them to resume. -func (b *Barrier) BlockUntil(count int) { - if count == 0 { - return - } - req := newBlockReq(count) - b.blockReqCh <- req - if count > 0 { - <-req.done - } -} - -// BlockAllReadersNoWait raises the barrier to -// an infinite number of waiters and returns immediately -// to the caller. -func (b *Barrier) BlockAllReadersNoWait() { - req := newBlockReq(-1) // -1 means block any number of readers. - b.blockReqCh <- req - // don't wait. <-req.done -} diff --git a/barrier_test.go b/barrier_test.go deleted file mode 100644 index 70fd4d311..000000000 --- a/barrier_test.go +++ /dev/null @@ -1,90 +0,0 @@ -// home https://github.com/glycerine/lmdb-go -// Copyright (c) 2020, the lmdb-go authors -// Copyright (c) 2015, Bryan Matsuo -// All rights reserved. - -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: - -// Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. - -// Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. - -// Neither the name of the author nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. - -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package pilosa - -import ( - "fmt" - "sync/atomic" - "testing" - "time" -) - -func TestBarrierHolds(t *testing.T) { - b := NewBarrier() - defer b.Close() - - released := int64(0) - - waiter := func(i int) { - b.WaitAtGate(i) - //vv("goro %v is released", i) - atomic.AddInt64(&released, 1) - } - - for i := 0; i < 3; i++ { - go waiter(i) - } - time.Sleep(time.Second) - r := atomic.SwapInt64(&released, 0) - if r != 3 { - panic("open barrier held back goro") - } - //vv("good: barrier started open") - - //seenAll := make(chan bool) - b.BlockAllReadersNoWait() - for i := 0; i < 3; i++ { - go waiter(i) - } - - time.Sleep(time.Second) - r = atomic.SwapInt64(&released, 0) - if r != 0 { - panic("bad: barrier did not hold back goro") - } - //vv("good: barrier of 4 did not release on 3") - go waiter(4) - - time.Sleep(time.Second) - r = atomic.SwapInt64(&released, 0) - if r != 0 { - panic(fmt.Sprintf("bad: barrier did not hold back goro, should wait for unblock. r = %v", r)) - } - - b.UnblockReaders() - - time.Sleep(time.Second) - r = atomic.SwapInt64(&released, 0) - if r != 4 { - panic("bad: unblock should have released 4 goro") - } - -} diff --git a/bluegreentx.go b/bluegreentx.go deleted file mode 100644 index 07c716437..000000000 --- a/bluegreentx.go +++ /dev/null @@ -1,1028 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "fmt" - "io" - "reflect" - "sync" - - "github.com/molecula/featurebase/v2/roaring" - txkey "github.com/molecula/featurebase/v2/short_txkey" - . "github.com/molecula/featurebase/v2/vprint" -) - -// blueGreenTx runs two Tx together and notices differences in their output. -// By convention, the 'b' Tx is the output that is returned to caller. -// -// Warning: DATA RACES are expected if RoaringTx is one side of the Tx pair. -// The checkDatabase() call will do reads of the fragments at Commit/Rollback, -// while the snapshotqueue may be doing writes. -// -// Do not run with go test -race and expect it to be race free with RoaringTx -// on one arm. -// -// Note: using the dbshard.go DBShard.mut RWMutex to begin and end -// both the A and B transactions atomically, we support a single importer and -// lots of readers running under blue-green transactions. Two writers a.k.a. two -// github ingests at once will deadlock eventually, but I think that may be asking -// for more than we want to test under blue-green, as it would require a bunch of -// test-only internal executor logic that could mess with the production path. -// So, for now, a limitation on blue green tests is that they be single -// writer/single importer going at once. -// -type blueGreenTx struct { - a Tx - b Tx // b's output is returned - - o Txo - as string - bs string - - types []txtype - hasRoaring bool - - // roaring will not create as many Tx (they are - // psuedo Tx anyway), espcially when deleting - // files. Return the non-roaring Sn if - // possible, by referencing useSnA. - useSnA bool - - idx *Index - - checker blueGreenChecker - mu sync.Mutex - rollbackOrCommitDone bool - - txf *TxFactory - - short bool // short Dump or long - - FullDump bool // else quieter, don't attemp Dump() if false. -} - -// blueGreenRegistry is used to force checking of (read) transactions -// before writes happen, if roaring is on one of the A/B branches. -// Because roaring won't have an MVCC view of the world. Writes to -// roaring will show up, while writes to the DB won't show up on -// readTx that have already started. -type blueGreenRegistry struct { - mu sync.Mutex - m map[int64]*blueGreenTx - types []txtype - hasRoaring bool -} - -// if we have raoring in the mix we cannot expect reads -// to match up, but otherwise do. -func newBlueGreenReg(types []txtype) *blueGreenRegistry { - - hasRoaring := false - if types[0] == roaringTxn || types[1] == roaringTxn { - hasRoaring = true - } - return &blueGreenRegistry{ - m: make(map[int64]*blueGreenTx), - types: types, - hasRoaring: hasRoaring, - } -} - -// add remembers the tx so we can check that -// all tx were finished before Close(). -func (b *blueGreenRegistry) add(c *blueGreenTx) { - b.mu.Lock() - defer b.mu.Unlock() - if c.useSnA { - b.m[c.a.Sn()] = c - } else { - b.m[c.b.Sn()] = c - } -} - -func (b *blueGreenRegistry) finishedTx(tx *blueGreenTx) { - b.mu.Lock() - defer b.mu.Unlock() - sn := tx.Sn() - delete(b.m, sn) - //vv("blueGreenRegistry deleted _sn_ %v", sn) - - // Note that a tx.o.dbs.Cleanup(tx) call should not be needed, - // because the individual tx will call cleanup themselves. -} - -func (b *blueGreenRegistry) Close() { - b.mu.Lock() - defer b.mu.Unlock() - if len(b.m) > 0 { - PanicOn(fmt.Sprintf("still have open/unchecked blueGreenTx: '%#v'", b.m)) - //AlwaysPrintf("still have unchecked blueGreenTx: '%#v'", b.m) - } -} - -func (txf *TxFactory) newBlueGreenTx(a, b Tx, idx *Index, o Txo) *blueGreenTx { - as := a.Type() - bs := b.Type() - c := &blueGreenTx{a: a, - b: b, - idx: idx, - as: as, - bs: bs, - txf: txf, - types: txf.types, - hasRoaring: txf.blueGreenReg.hasRoaring, - short: true, - } - - if c.types[1] == roaringTxn { - c.useSnA = true - } - //vv("newBlueGreenTx with a.sn=%v with o.Shard=%v", c.Sn(), int(o.Shard)) - - c.checker.c = c - c.o = o - txf.blueGreenReg.add(c) - return c -} - -var _ Tx = (*blueGreenTx)(nil) - -func (c *blueGreenTx) Type() string { - return c.a.Type() + "_" + c.b.Type() -} - -var blueGreenTxDumpMut sync.Mutex - -func (c *blueGreenTx) Dump(short bool, shard uint64) { - - if !c.FullDump { - return - } - - blueGreenTxDumpMut.Lock() - defer blueGreenTxDumpMut.Unlock() - fmt.Printf("%v blueGreenTx.Dump ============== \n", FileLine(2)) - fmt.Printf("A(%v) Dump:\n", c.as) - c.a.Dump(short, shard) - fmt.Printf("B(%v) Dump:\n", c.bs) - c.b.Dump(short, shard) - - if !short { - fmt.Printf("dbPerShard.DumpAll(): idx=%p\n", c.idx) - c.idx.holder.txf.dbPerShard.DumpAll() - } -} - -func (c *blueGreenTx) Readonly() bool { - a := c.a.Readonly() - b := c.b.Readonly() - if a != b { - PanicOn(fmt.Sprintf("Readonly difference, a=%v, but b =%v", a, b)) - } - return b -} - -// for now we just return B's list, since this is involved in -// holder Open which can happen before any blue-green is done; -// in fact this is instrumental in setting up the sync from -// green to blue. -func (c *blueGreenTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvB []txkey.FieldView, errB error) { - fvB, errB = c.b.GetSortedFieldViewList(idx, shard) - return -} - -func (c *blueGreenTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { - c.checker.see(index, field, view, shard) - // can't really do simultaneous iteration on A and B, so punt and - // just give back B. - return c.b.NewTxIterator(index, field, view, shard) -} - -func (c *blueGreenTx) Pointer() string { - return fmt.Sprintf("%p", c) -} - -func (c *blueGreenTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - c.checker.see(index, field, view, shard) - c.a.IncrementOpN(index, field, view, shard, changedN) - c.b.IncrementOpN(index, field, view, shard, changedN) -} - -// compareTxState is called for the first Commit or Rollback a blueGreenTx sees. -func (c *blueGreenTx) compareTxState(index, field, view string, shard uint64) { - if c.o.blueGreenOff { - return - } - here := fmt.Sprintf("%v/%v/%v/%v", index, field, view, shard) - //vv("compareTxState here = '%v', _sn_ %v gid=%v", here, c.Sn(), curGID()) - aIter, aFound, aErr := c.a.ContainerIterator(index, field, view, shard, 0) - bIter, bFound, bErr := c.b.ContainerIterator(index, field, view, shard, 0) - if aErr == nil || aIter != nil { - defer aIter.Close() - } - if bErr == nil || bIter != nil { - defer bIter.Close() - } - - if aFound != bFound { - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) ContainerIterator had aFound=%v, but B(%v) had bFound=%v; at '%v'", here, c.as, aFound, c.bs, bFound, Stack())) - } - - if aErr != nil || bErr != nil { - if aErr != nil && bErr != nil { - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) reported err '%v'; B(%v) reported err '%v' at %v", here, c.as, aErr, c.bs, bErr, Stack())) - } - if aErr != nil { - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) reported err %v at %v; but B(%v) did not", here, c.as, aErr, c.bs, Stack())) - } - if bErr != nil { - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: B(%v) reported err %v at %v; but A(%v) did not", here, c.bs, bErr, c.as, Stack())) - } - } - for aIter.Next() { - aKey, aValue := aIter.Value() - - if !bIter.Next() { - AlwaysPrintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, dump to follow, Stack=\n %v\n\n and here is dump:", here, c.as, aKey, c.bs, Stack()) - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) didn't, at %v", here, c.as, aKey, c.bs, Stack())) - } - bKey, bValue := bIter.Value() - if bKey != aKey { - AlwaysPrintf("problem in caller %v", Caller(2)) - c.Dump(c.short, shard) - PanicOn(fmt.Sprintf("compareTxState[%v]: A(%v) found key %v, B(%v) found %v, at %v", here, c.as, aKey, c.bs, bKey, Stack())) - } - if err := aValue.BitwiseCompare(bValue); err != nil { - c.Dump(c.short, shard) - //vv("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at Stack=%v", here, aKey, err, c.as, c.bs, Stack()) - PanicOn(fmt.Sprintf("compareTxState[%v]: key %v differs: %v; A=%v; B=%v; at Stack=%v", here, aKey, err, c.as, c.bs, Stack())) - } - //vv("successfully matched aKey(%v)='%v' and bKey(%v)='%v'", c.as, aKey, c.bs, bKey) - } - // end checking everything in A, but does B have more? - if bIter.Next() { - AlwaysPrintf("bIter has more than it should. problem in caller %v. _sn_ %v", Caller(2), c.Sn()) - c.Dump(c.short, shard) - bKey, _ := bIter.Value() - PanicOn(fmt.Sprintf("compareTxState[%v]: B(%v) found key %v, A(%v) didn't, (a.sn=%v) (b.sn=%v) at %v", here, c.bs, bKey, c.as, c.a.Sn(), c.b.Sn(), Stack())) - } - //vv("done without problem. compareTxState here = '%v', _sn_ %v gid=%v", here, c.Sn(), curGID()) -} - -func (c *blueGreenTx) checkDatabase() { - if c.o.blueGreenOff { - return - } - if c.hasRoaring && !c.o.Write { - // With roaring on one arm, we only check the we are A/B - // consistent after every write. - // - // Ideally reads can only see that consitent state, and don't need - // to be checked themselves-- but we do try if both A and B - // are transactional. Sketch of proof by induction that - // write checking should, theoretically, suffice: - // Starting with zero data, if we have agreement in both A/B - // database state after each write, then - // because there is only ever a single - // writer (for LMDB/RBF), we should always have the same - // data state between A and B as long as every prior - // A/B check of the serialized writes suceeded. - // - // This avoids a key problem we discovered when A/B checking reads - // with roaring on one arm. - // The MVCC of the transactional engines means that reads that - // start before a write commit will look very different - // when comparing to roaring's non-transactional state. - return - } - - c.checker.mu.Lock() - defer c.checker.mu.Unlock() - if c.checker.checkDone { - return // idemopotent. checkDatabase can be called twice. Only the first does the checks. - } - c.checker.checkDone = true - - for index, fields := range c.checker.seen() { - for field, views := range fields { - for view, shards := range views { - for shard := range shards { - c.compareTxState(index, field, view, shard) - } - } - } - } -} - -func (c *blueGreenTx) IsDone() bool { - return c.b.IsDone() -} - -func (c *blueGreenTx) Rollback() { - c.mu.Lock() - defer c.mu.Unlock() - if c.rollbackOrCommitDone { - return // avoid using discarded tx for Dump, which will PanicOn. - } - c.rollbackOrCommitDone = true - - if c.o.Write { - c.checkDatabase() - } - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Rollback() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - //vv("blueGreenTx.Rollback() about to call (%v) a.Rollback()", c.as) - c.a.Rollback() - //vv("blueGreenTx.Rollback() about to call (%v) b.Rollback()", c.bs) - c.b.Rollback() - //vv("blueGreenTx.Rollback() done. bgtx p=%p", c) - - c.txf.blueGreenReg.finishedTx(c) -} - -func (c *blueGreenTx) Commit() error { - c.mu.Lock() - defer c.mu.Unlock() - - if c.rollbackOrCommitDone { - return nil - } - //vv("blueGreenTx.Commit() called. bgtx p=%p", c) - c.rollbackOrCommitDone = true - if c.o.Write { - if !c.o.blueGreenOff { - c.checkDatabase() - } - } - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Commit() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - errA := c.a.Commit() - _ = errA - errB := c.b.Commit() - - compareErrors(errA, errB) - c.txf.blueGreenReg.finishedTx(c) - return errB -} - -func (c *blueGreenTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see RoaringBitmap() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.RoaringBitmap(index, field, view, shard) - _, _ = a, errA - b, errB := c.b.RoaringBitmap(index, field, view, shard) - if !c.o.blueGreenOff { - compareErrors(errA, errB) - - slcA := a.Slice() - slcB := b.Slice() - if !reflect.DeepEqual(slcA, slcB) { - PanicOn("blueGreenTx.RoaringBitmap() returning different roaring.Bitmaps!") - } - } - return b, errB -} - -func (c *blueGreenTx) Container(index, field, view string, shard uint64, key uint64) (ct *roaring.Container, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Container() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.Container(index, field, view, shard, key) - b, errB := c.b.Container(index, field, view, shard, key) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - err = a.BitwiseCompare(b) - PanicOn(err) - } - return b, errB -} - -func (c *blueGreenTx) PutContainer(index, field, view string, shard uint64, key uint64, rc *roaring.Container) error { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see PutContainer() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - errA := c.a.PutContainer(index, field, view, shard, key, rc) - errB := c.b.PutContainer(index, field, view, shard, key, rc) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return errB -} - -func (c *blueGreenTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { - c.checker.see(index, field, view, shard) - - // these are the first port of call for debugging, so we leave them in. - // ================== begin save comments. - //c.checkDatabase() - ////vv("got past database check at TOP of ImportRoaringBits") - //c.Dump(c.short, shard) - ////vv("done with top dump; clear=%v", clear) - // ================== end save comments. - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - - // remember where the iterator started, so we can replay it a second time. - rit2 := rit.Clone() - PanicOn(err) - - changedA, rowSetA, errA := c.a.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) - changedB, rowSetB, errB := c.b.ImportRoaringBits(index, field, view, shard, rit2, clear, log, rowSize, data) - - if !c.o.blueGreenOff { - - if len(data) == 0 { - // okay to check! otherwise we are in the fragment.fillFragmentFromArchive - // case where we know that RoaringTx.ImportRoaringBits changed and rowSet will - // be inaccurate. - if changedA != changedB { - PanicOn(fmt.Sprintf("changedA = %v, but changedB = %v", changedA, changedB)) - } - if len(rowSetA) != len(rowSetB) { - PanicOn(fmt.Sprintf("rowSetA = %#v, but rowSetB = %#v", rowSetA, rowSetB)) - } - for k, va := range rowSetA { - vb, ok := rowSetB[k] - if !ok { - PanicOn(fmt.Sprintf("diff on key '%v': present in rowSetA, but not in rowSet B. rowSetA = %#v, but rowSetB = %#v", k, rowSetA, rowSetB)) - } - if va != vb { - PanicOn(fmt.Sprintf("diff on key '%v', rowSetA has value '%v', but rowSetB has value '%v'", k, va, vb)) - } - } - } - compareErrors(errA, errB) - c.checkDatabase() - } - return changedB, rowSetB, errB -} - -func (c *blueGreenTx) RemoveContainer(index, field, view string, shard uint64, key uint64) error { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see RemoveContainer() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - errA := c.a.RemoveContainer(index, field, view, shard, key) - errB := c.b.RemoveContainer(index, field, view, shard, key) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return errB -} - -func (c *blueGreenTx) UseRowCache() bool { - // avoid cross-talk between our two implementations - // by never allowing either to use the row cache. - return false -} - -var _ = (&blueGreenTx{}).isIn // happy linter - -func (c *blueGreenTx) isIn(index, field, view string, shard uint64, ckey uint64) (r []bool) { - r = make([]bool, 2) - inA, errA := c.a.Contains(index, field, view, shard, ckey) - PanicOn(errA) - inB, errB := c.b.Contains(index, field, view, shard, ckey) - PanicOn(errB) - r[0] = inA - r[1] = inB - return -} - -func (c *blueGreenTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - c.checker.see(index, field, view, shard) - //vv("blueGreenTx) Add(index=%v, field=%v, view=%v, shard=%v", index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Add() PanicOn '%v' for index='%v', field='%v', view='%v', shard='%v' at '%v'", r, index, field, view, shard, Stack()) - PanicOn(r) - } - }() - - // must copy a before calling Add(), since RoaringTx.Add() uses roaring.DirectAddN() - // which modifies the input array a. - a2 := make([]uint64, len(a)) - copy(a2, a) - - ach, errA := c.a.Add(index, field, view, shard, a...) - _, _ = ach, errA - - bch, errB := c.b.Add(index, field, view, shard, a2...) - - if !c.o.blueGreenOff { - - if ach != bch { - PanicOn(fmt.Sprintf("Add() difference, ach=%v, but bch=%v; errA='%v'; errB='%v'", ach, bch, errA, errB)) - } - compareErrors(errA, errB) - } - - return bch, errB -} - -func compareErrors(errA, errB error) { - switch { - case errA == nil && errB == nil: - // OK - case errA == nil: - PanicOn(fmt.Sprintf("errA is nil, but errB = %#v", errB)) - case errB == nil: - PanicOn(fmt.Sprintf("errB is nil, but errA = %#v", errA)) - default: - ae := errA.Error() - be := errB.Error() - if ae != be { - PanicOn(fmt.Sprintf("errA is '%v', but errB is '%v'", ae, be)) - } - } -} - -func (c *blueGreenTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Remove() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - ach, errA := c.a.Remove(index, field, view, shard, a...) - _, _ = ach, errA - bch, errB := c.b.Remove(index, field, view, shard, a...) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return bch, errB -} - -func (c *blueGreenTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Contains() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - ax, errA := c.a.Contains(index, field, view, shard, key) - _, _ = ax, errA - bx, errB := c.b.Contains(index, field, view, shard, key) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return bx, errB -} - -func (c *blueGreenTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see ContainerIterator() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - - ait, afound, errA := c.a.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) - _, _, _ = ait, afound, errA - - bit, bfound, errB := c.b.ContainerIterator(index, field, view, shard, firstRoaringContainerKey) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - - if errB != nil { - // RoaringTx can return an iterator and an error, so be sure Close it we have it. - if ait != nil { - ait.Close() - } - if bit != nil { - bit.Close() - } - return nil, bfound, errB - } - if errA != nil { - // RoaringTx can return an iterator and an error, so be sure Close it we have it. - if ait != nil { - ait.Close() - } - } - - // INVAR: errA == errB == nil - bgi := NewBlueGreenIterator(c, ait, bit) - return bgi, bfound, errB -} - -func (tx *blueGreenTx) GetFieldSizeBytes(index, field string) (uint64, error) { - return 0, nil -} - -func NewBlueGreenIterator(tx *blueGreenTx, ait, bit roaring.ContainerIterator) *blueGreenIterator { - return &blueGreenIterator{ - tx: tx, - as: tx.as, - bs: tx.bs, - ait: ait, - bit: bit, - } -} - -type blueGreenIterator struct { - tx *blueGreenTx - as string - bs string - - ait roaring.ContainerIterator - bit roaring.ContainerIterator -} - -func (bgi *blueGreenIterator) Next() bool { - na := bgi.ait.Next() - nb := bgi.bit.Next() - if na != nb { - PanicOn(fmt.Sprintf("na=%v(%v) != nb(%v)=%v", na, bgi.as, bgi.bs, nb)) - } - return nb -} - -func (bgi *blueGreenIterator) Value() (uint64, *roaring.Container) { - ka, ca := bgi.ait.Value() - kb, cb := bgi.bit.Value() - - if !bgi.tx.o.blueGreenOff { - if ka != kb { - PanicOn(fmt.Sprintf("ka=%v != kb=%v", ka, kb)) - } - err := ca.BitwiseCompare(cb) - PanicOn(err) - } - return kb, cb -} -func (bgi *blueGreenIterator) Close() { - bgi.ait.Close() - bgi.bit.Close() -} - -// ForEach is read-only on the database, and so we only pass through to B. -// Avoids the side-effects of calling fn too many times, which can cause serious false alarms. -func (c *blueGreenTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see ForEach() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.ForEach(index, field, view, shard, fn) - -} - -// ForEachRange cannot change the database, and we also can't control -// the side effects of the fn() calls. So we only pass through to B, not A. -// No checker.see() is needed as well, because we are read-only. -func (c *blueGreenTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { - - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see ForEachRange() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - - // calling fn will have side effects; can only call it the right number of times. - // so can't do this. - // errA := c.a.ForEachRange(index, field, view, shard, start, end, fn) - return c.b.ForEachRange(index, field, view, shard, start, end, fn) -} - -func (c *blueGreenTx) Count(index, field, view string, shard uint64) (uint64, error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Count() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.Count(index, field, view, shard) - _, _ = a, errA - b, errB := c.b.Count(index, field, view, shard) - _, _ = b, errB - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return b, errB -} - -func (c *blueGreenTx) Max(index, field, view string, shard uint64) (uint64, error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Max() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.Max(index, field, view, shard) - _, _ = a, errA - b, errB := c.b.Max(index, field, view, shard) - _, _ = b, errB - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return b, errB -} - -func (c *blueGreenTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Min() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - amin, afound, errA := c.a.Min(index, field, view, shard) - _, _, _ = amin, afound, errA - bmin, bfound, errB := c.b.Min(index, field, view, shard) - _, _, _ = bmin, bfound, errB - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return bmin, bfound, errB -} - -func (c *blueGreenTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - errA := c.a.UnionInPlace(index, field, view, shard, others...) - errB := c.b.UnionInPlace(index, field, view, shard, others...) - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - return errB -} - -func (c *blueGreenTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - c.Dump(c.short, shard) - AlwaysPrintf("see CountRange() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.CountRange(index, field, view, shard, start, end) - b, errB := c.b.CountRange(index, field, view, shard, start, end) - - if !c.o.blueGreenOff { - if a != b { - PanicOn(fmt.Sprintf("a(%v) = %v, but b(%v) = %v", c.as, a, c.bs, b)) - } - - compareErrors(errA, errB) - } - return b, errB -} - -func (c *blueGreenTx) OffsetRange(index, field, view string, shard, offset, start, end uint64) (other *roaring.Bitmap, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see OffsetRange() on _sn_ %v, PanicOn '%v' at '%v'", c.Sn(), r, Stack()) - PanicOn(r) - } - }() - a, errA := c.a.OffsetRange(index, field, view, shard, offset, start, end) - b, errB := c.b.OffsetRange(index, field, view, shard, offset, start, end) - - if !c.o.blueGreenOff { - - err = roaringBitmapDiff(a, b) - if err != nil { - c.Dump(false, shard) - PanicOn(fmt.Errorf("on _sn_ %v OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) err: %v", c.Sn(), index, field, view, int(shard), offset, start, end, err)) - } - compareErrors(errA, errB) - } - - return b, errB -} - -func (c *blueGreenTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - c.checker.see(index, field, view, shard) - defer func() { - if r := recover(); r != nil { - c.Dump(c.short, shard) - AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - - rcA, szA, errA := c.a.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) - rcB, szB, errB := c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) - - if !c.o.blueGreenOff { - compareErrors(errA, errB) - } - - // We are seeing Roaring vs Badger size differences on - // server/ test TestClusterResize_AddNode/ContinuousShards, - // so turn off the szA vs szB checks and MutliReaderB use. But keep them if we want to - // check RBF vs Badger for byte-for-byte compatiblity (we - // suspect the ops log or optimized bitmaps are accounting for the difference). - sizeMustMatch := false // !c.hasRoaring - if c.o.blueGreenOff { - sizeMustMatch = false - } - if sizeMustMatch { - if szA != szB { - PanicOn(fmt.Sprintf("szA(%v) = %v, but szB(%v) = %v; fragmentPathForRoaring='%v'", c.as, szA, c.bs, szB, fragmentPathForRoaring)) - } - return &MultiReaderB{a: rcA, b: rcB}, szB, errB - } else { - // one db won't get data if we do - //return &MultiReaderB{a: rcA, b: rcB, allowSizeVariation: true}, szB, errB - _, _ = szA, errA - rcA.Close() - return rcB, szB, errB - } -} - -func (c *blueGreenTx) Group() *TxGroup { - return c.b.Group() -} - -func (c *blueGreenTx) Options() Txo { - return c.b.Options() -} - -// Sn retreives the serial number of the Tx. -func (c *blueGreenTx) Sn() int64 { - asn := c.a.Sn() - bsn := c.b.Sn() - - if c.useSnA { - return asn - } - return bsn -} - -func (c *blueGreenTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { - return GenericApplyFilter(c, index, field, view, shard, ckey, filter) -} - -// MultiReaderB is returned by RoaringBitmapReader. It verifies -// that identical byte streams are read from its two members. -type MultiReaderB struct { - a io.ReadCloser - b io.ReadCloser - - allowSizeVariation bool -} - -// Read implements the standard io.Reader method. It panics -// if "a" and "b" have even one byte different in their reads. -func (m *MultiReaderB) Read(p []byte) (nB int, errB error) { - nB, errB = m.b.Read(p) - p2 := make([]byte, nB) - - // read (and discard after comparing for equality) the exact same amount from A. - - // ReadAtLeast reads from r into buf until it has read at least - // min bytes. It returns the number of bytes copied and an error - // if fewer bytes were read. The error is EOF only if no bytes - // were read. If an EOF happens after reading fewer than min bytes, - // ReadAtLeast returns ErrUnexpectedEOF. If min is greater than - // the length of buf, ReadAtLeast returns ErrShortBuffer. On - // return, n >= min if and only if err == nil. If r returns - // an error having read at least min bytes, the error is dropped. - nA, errA := io.ReadAtLeast(m.a, p2, nB) - - if !m.allowSizeVariation { - if errA == io.ErrUnexpectedEOF { - PanicOn(fmt.Sprintf("MultiReaderB got ErrUnexpectedEOF: read %v bytes from B, but could only read %v bytes for A", nB, nA)) - } - if nA != nB { - PanicOn(fmt.Sprintf("MultiReaderB read %v bytes from B, but could only read %v bytes for A", nB, nA)) - } - cmp := bytes.Compare(p[:nB], p2[:nB]) - if cmp != 0 { - PanicOn(fmt.Sprintf("MultiReaderB reads p and p2 (cmp= %v) differed.", cmp)) - } - } - return -} - -func (m *MultiReaderB) Close() error { - m.a.Close() - return m.b.Close() -} - -// blueGreenChecker is used -type blueGreenChecker struct { - visited map[string]map[string]map[string]map[uint64]struct{} - - c *blueGreenTx - - // lock mu when using visited. - // otherwise concurrent map writes on TestAPI_Import/RowIDColumnKey - mu sync.Mutex - - checkDone bool -} - -// see would mark a thing as seen. -func (b *blueGreenChecker) see(index, field, view string, shard uint64) { - // keep this next Printf. Useful to see the sequence of Tx operations. - //fmt.Printf("blueGreenTx.%v on index='%v' shard=%v\n", Caller(1), index, shard) - - if !b.c.o.Write { - return - } - - b.mu.Lock() - defer b.mu.Unlock() - - if b.visited == nil { - b.visited = make(map[string]map[string]map[string]map[uint64]struct{}) - } - var visitedIdx map[string]map[string]map[uint64]struct{} - var visitedField map[string]map[uint64]struct{} - var visitedView map[uint64]struct{} - - if visitedIdx = b.visited[index]; visitedIdx == nil { - visitedIdx = make(map[string]map[string]map[uint64]struct{}) - b.visited[index] = visitedIdx - } - if visitedField = visitedIdx[field]; visitedField == nil { - visitedField = make(map[string]map[uint64]struct{}) - visitedIdx[field] = visitedField - } - if visitedView = visitedField[view]; visitedView == nil { - visitedView = make(map[uint64]struct{}) - visitedField[view] = visitedView - } - visitedView[shard] = struct{}{} -} - -// seen reports the things it has seen, exactly once so -// that Rollback can be called after Commit without repeating -// the check. -func (b *blueGreenChecker) seen() map[string]map[string]map[string]map[uint64]struct{} { - return b.visited -} diff --git a/bluegreentx_test.go b/bluegreentx_test.go deleted file mode 100644 index 3a365800b..000000000 --- a/bluegreentx_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "context" - "io" - "io/ioutil" - "os" - "strings" - "testing" - - cryrand "crypto/rand" - - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck -) - -var _ = context.Background -var _ = os.Open -var _ = strings.Split - -func TestMultiReaderB(t *testing.T) { - // MultiReaderB should read identical chunks of bytes from both its "a" and "b" - // member io.Readers, else it should panic. This should hold for - // varying sizes of inputs. - - for n := 1 << 5; n < (1 << 18); n = n*2 - 13 { - src := io.LimitReader(cryrand.Reader, int64(n)) - - a := make([]byte, n) - nr := 0 - for nr < n { - na, err := src.Read(a) - PanicOn(err) - nr += na - } - if nr != n { - panic("short read") - } - - b := make([]byte, n) - copy(b, a) - if !bytes.Equal(a, b) { - panic("test prep failed") - } - - m := &MultiReaderB{ - a: ioutil.NopCloser(bytes.NewBuffer(a)), - b: ioutil.NopCloser(bytes.NewBuffer(b)), - } - - // should not trigger the internal panic of MultiReadB - ncp, err := io.Copy(ioutil.Discard, m) - PanicOn(err) - if ncp != int64(n) { - panic("short copy") - } - - for victim := 0; victim < n; victim += 7 { - - copy(b, a) - if victim%2 == 0 { - // corrupt b - b[victim] = (b[victim] + 1) % 255 - } else { - // corrupt a - a[victim] = (a[victim] + 1) % 255 - } - m = &MultiReaderB{ - a: ioutil.NopCloser(bytes.NewBuffer(a)), - b: ioutil.NopCloser(bytes.NewBuffer(b)), - } - helperShouldPanicOnCopy(m) - } - } -} - -func helperShouldPanicOnCopy(m *MultiReaderB) { - // differences in bytes read should be noticed - defer func() { - r := recover() - if r == nil { - panic("expected panic on byte difference but didn't see it") - } - }() - _, _ = io.Copy(ioutil.Discard, m) -} diff --git a/bolt.go b/bolt.go deleted file mode 100644 index 12435abef..000000000 --- a/bolt.go +++ /dev/null @@ -1,1612 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "math" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/molecula/featurebase/v2/hash" - "github.com/molecula/featurebase/v2/roaring" - "github.com/molecula/featurebase/v2/storage" - . "github.com/molecula/featurebase/v2/vprint" - - // On Bolt only, we still use the long txkey, because - // this allows Max() to work readily. - // - "github.com/molecula/featurebase/v2/short_txkey" - txkey "github.com/molecula/featurebase/v2/txkey" - "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" -) - -const isDebugRun = false - -// boltRegistrar facilitates shutdown -// of all the bolt databases started under -// tests. Its needed because most tests don't cleanup -// the *Index(es) they create. But we still -// want to shutdown boltDB goroutines -// after tests run. -// -// It also allows opening the same path twice to -// result in sharing the same open database handle, and -// thus the same transactional guarantees. -// -type boltRegistrar struct { - mu sync.Mutex - mp map[*BoltWrapper]bool - - path2db map[string]*BoltWrapper -} - -func (r *boltRegistrar) Size() int { - r.mu.Lock() - defer r.mu.Unlock() - nmp := len(r.mp) - npa := len(r.path2db) - if nmp != npa { - panic(fmt.Sprintf("nmp=%v, vs npa=%v", nmp, npa)) - } - return nmp -} - -var globalBoltReg *boltRegistrar = newBoltTestRegistrar() - -var globalNextTxSnBolt int64 - -func newBoltTestRegistrar() *boltRegistrar { - - return &boltRegistrar{ - mp: make(map[*BoltWrapper]bool), - path2db: make(map[string]*BoltWrapper), - } -} - -// register each bolt created under tests, so we -// can clean them up. This is called by openBoltWrapper() while -// holding the r.mu.Lock, since it needs to atomically -// check the registry and make a new instance only -// if one does not exist for its path, and otherwise -// return the existing instance. -func (r *boltRegistrar) unprotectedRegister(w *BoltWrapper) { - r.mp[w] = true - r.path2db[w.path] = w -} - -// unregister removes w from r -func (r *boltRegistrar) unregister(w *BoltWrapper) { - r.mu.Lock() - delete(r.mp, w) - delete(r.path2db, w.path) - r.mu.Unlock() -} - -func DumpAllBolt() { - short := true - globalBoltReg.mu.Lock() - defer globalBoltReg.mu.Unlock() - for w := range globalBoltReg.mp { - AlwaysPrintf("this bolt path='%v' has: \n%v\n", w.path, w.StringifiedBoltKeys(nil, short)) - } -} - -// openBoltDB opens the database in the bpath directoy -// without deleting any prior content. Any BoltDB -// database directory will have the "-bolt" suffix. -// -// openBoltDB will check the registry and make a new instance only -// if one does not exist for its bpath. Otherwise it returns -// the existing instance. This insures only one boltDB -// per bpath in this pilosa node. -func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storage.Config) (DBWrapper, error) { - r.mu.Lock() - defer r.mu.Unlock() - w, ok := r.path2db[path] - if ok { - // creates the effect of having only one bolt open per pilosa node. - return w, nil - } - // otherwise, make a new bolt and store it in globalBoltReg - - dir := filepath.Dir(path) - if !DirExists(path) { - PanicOn(os.MkdirAll(dir, 0755)) - } - fsyncEnabled := true - if cfg != nil { - fsyncEnabled = cfg.FsyncEnabled - } - - db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !fsyncEnabled}) - if err != nil { - return nil, errors.Wrapf(err, fmt.Sprintf("open bolt path '%v'", path)) - } - - // docs on fsync from https://godoc.org/github.com/etcd-io/bbolt - // - // Setting the NoSync flag will cause the database to skip fsync() - // calls after each commit. This can be useful when bulk loading data - // into a database and you can restart the bulk load in the event of - // a system failure or database corruption. Do not set this flag for - // normal use. - // - // If the package global IgnoreNoSync constant is true, this value is - // ignored. See the comment on that constant for more details. - // - // THIS IS UNSAFE. PLEASE USE WITH CAUTION. - // NoSync bool - - // When true, skips syncing freelist to disk. This improves the database - // write performance under normal operation, but requires a full database - // re-sync during recovery. - // NoFreelistSync bool - - if cfg != nil && !cfg.FsyncEnabled { - db.NoSync = true - db.NoFreelistSync = true - } else { - // default to using fsync on bolt. - db.NoSync = false - db.NoFreelistSync = false - } - - err = db.Update(func(tx *bolt.Tx) (err error) { - _, err = tx.CreateBucketIfNotExists(bucketCT) - return - }) - if err != nil { - return nil, errors.Wrapf(err, fmt.Sprintf("create bolt bucket '%v' in path '%v'", string(bucketCT), path)) - } - - name := filepath.Base(path) - w = &BoltWrapper{ - name: name, - db: db, - reg: r, - path: path, - doAllocZero: doAllocZero, - openTx: make(map[*BoltTx]bool), - - DeleteEmptyContainer: true, - fsyncEnabled: cfg.FsyncEnabled, - } - r.unprotectedRegister(w) - - return w, nil -} - -func (w *BoltWrapper) Path() string { - return w.path -} - -func (w *BoltWrapper) HasData() (has bool, err error) { - - tx, err := w.NewTx(!writable, "", Txo{}) - if err != nil { - return false, errors.Wrap(err, "HasData NewTx") - } - defer tx.Rollback() - - bi := NewBoltIterator(tx.(*BoltTx), nil) - defer bi.Close() - - for bi.Next() { - return true, nil - } - return false, nil -} - -func (w *BoltWrapper) CleanupTx(tx Tx) { - // inlined into Rollback and Commit, so this is a no-op, just here to satisfy the interface. -} -func (w *BoltWrapper) CloseDB() error { - w.muDb.Lock() - defer w.muDb.Unlock() - w.closed = true - return w.db.Close() -} -func (w *BoltWrapper) OpenDB() error { - w.muDb.Lock() - defer w.muDb.Unlock() - db, err := bolt.Open(w.path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !w.fsyncEnabled}) - if err != nil { - return err - } - w.db = db - w.closed = false - return nil -} - -func (tx *BoltTx) IsDone() (done bool) { - return atomic.LoadInt64(&tx.unlocked) == 1 -} - -func (w *BoltWrapper) OpenListString() (r string) { - - list := w.listopen() - if len(list) == 0 { - return "" - } - for i, ltx := range list { - if ltx.o.Write { - r += fmt.Sprintf("[%v]write: _sn_ %v %v, \n", i, ltx.sn, ltx.o) - } else { - r += fmt.Sprintf("[%v]read : _sn_ %v %v, \n", i, ltx.sn, ltx.o) - } - } - return -} - -func (w *BoltWrapper) listopen() (slc []*BoltTx) { - w.muDb.Lock() - for v := range w.openTx { - slc = append(slc, v) - } - w.muDb.Unlock() - return -} - -func (w *BoltWrapper) OpenSnList() (slc []int64) { - w.muDb.Lock() - for v := range w.openTx { - slc = append(slc, v.sn) - } - w.muDb.Unlock() - return -} - -// DeleteIndex deletes all the containers associated with -// the named index from the bolt database. -func (w *BoltWrapper) DeleteIndex(indexName string) error { - - // We use the apostrophie rune `'` to locate the end of the - // index name in the key prefix, so we cannot allow indexNames - // themselves to contain apostrophies. - if strings.Contains(indexName, "/") { - return fmt.Errorf("error: bad indexName `%v` in BoltWrapper.DeleteIndex() call: indexName cannot contain '/'", indexName) - } - prefix := txkey.IndexOnlyPrefix(indexName) - return w.DeletePrefix(prefix) -} - -// statically confirm that BoltTx satisfies the Tx interface. -var _ Tx = (*BoltTx)(nil) - -// BoltWrapper provides the NewTx() method. -type BoltWrapper struct { - db *bolt.DB - - muDb sync.Mutex - - path string - name string - - // track our registrar for Close / goro leak reporting purposes. - reg *boltRegistrar - - // make BoltWrapper.Close() idempotent, avoiding panic on double Close() - closed bool - - // doAllocZero sets the corresponding flag on all new BoltTx. - // When doAllocZero is true, we zero out any data from bolt - // after transcation commit and rollback. This simulates - // what would happen if we were to use the mmap-ed data - // from bolt directly. Currently we copy by default for - // safety because otherwise TestAPI_ImportColumnAttrs sees - // corrupted data. - doAllocZero bool - - DeleteEmptyContainer bool - fsyncEnabled bool // for tracking whether our initial config wanted fsync on - - openTx map[*BoltTx]bool -} - -func (w *BoltWrapper) SetHolder(h *Holder) { - // don't need it at the moment - //w.h = h -} - -// NewTxWRITE lets us see in the callstack dumps where the WRITE tx are. -// Can't have more than one active write per database, so the -// 2nd one will block until the first finishes. -func (w *BoltWrapper) NewTxWRITE() (*bolt.Tx, error) { - boltTxn, err := w.db.Begin(true) - if err != nil { - if w.db == nil || w.IsClosed() { - return nil, fmt.Errorf("cannot call NewTxWRITE() on closed Bolt database: '%v'", err) - } - return nil, err - } - return boltTxn, nil -} - -// NewTxREAD lets us see in the callstack dumps where the READ tx are. -func (w *BoltWrapper) NewTxREAD() (*bolt.Tx, error) { - boltTxn, err := w.db.Begin(false) - if err != nil { - if w.db == nil || w.IsClosed() { - return nil, fmt.Errorf("cannot call NewTxREAD() on closed Bolt database: '%v'", err) - } - return nil, err - } - return boltTxn, nil -} - -// NewTx produces Bolt based transactions. If -// the transaction will modify data, then the write flag must be true. -// Read-only queries should set write to false, to allow more concurrency. -// Methods on a BoltTx are thread-safe, and can be called from -// different goroutines. -// -// initialIndexName is optional. It is set by the TxFactory from the Txo -// options provided at the Tx creation point. It allows us to recognize -// and isolate cross-index queries more quickly. It can always be empty "" -// but when set is highly useful for debugging. It has no impact -// on transaction behavior. -// -func (w *BoltWrapper) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { - - sn := atomic.AddInt64(&globalNextTxSnBolt, 1) - - ////vv("bolt new tx _sn_ %v; openTx='%v', stack \n%v", sn, w.OpenListString(), stack()) - ////vv("bolt new (write=%v, shard=%v) tx _sn_ %v; openTx='%v'", write, o.Shard, sn, w.OpenListString()) - - var boltTxn *bolt.Tx - if write { - // see the WRITE tx on the callstack. - boltTxn, err = w.NewTxWRITE() - if err != nil { - return nil, err - } - } else { - // see the READ tx on the callstack. - boltTxn, err = w.NewTxREAD() - if err != nil { - return nil, err - } - } - - ltx := &BoltTx{ - sn: sn, - write: write, - tx: boltTxn, - Db: w, - frag: o.Fragment, - doAllocZero: w.doAllocZero, - initialIndexName: initialIndexName, - DeleteEmptyContainer: w.DeleteEmptyContainer, - o: o, - gid: curGID(), - } - tx = ltx - - if isDebugRun { - w.muDb.Lock() - w.openTx[ltx] = true - w.muDb.Unlock() - } - return -} - -// Close shuts down the Bolt database. -func (w *BoltWrapper) Close() (err error) { - w.muDb.Lock() - defer w.muDb.Unlock() - if !w.closed { - if isDebugRun { - // complain if there are still Tx in flight, b/c otherwise we will see - // the somewhat mysterious 'panic: should not be in ReadSlot.free() with slot still owned by gid=107043; refCount=1' - if len(w.openTx) > 0 { - AlwaysPrintf("error: cannot close BoltWrapper with Tx still in flight.") - return - } - } - w.reg.unregister(w) - w.closed = true - return w.db.Close() - } - return nil -} - -func (w *BoltWrapper) IsClosed() (closed bool) { - w.muDb.Lock() - closed = w.closed - w.muDb.Unlock() - return -} - -// BoltTx wraps a bolt.Tx and provides the Tx interface -// method implementations. -// The methods on BoltTx are thread-safe, and can be called -// from different goroutines. -type BoltTx struct { - - // mu serializes bolt operations on this single txn instance. - mu sync.Mutex - sn int64 // serial number - - write bool - Db *BoltWrapper - tx *bolt.Tx - frag *fragment - - opcount int - - //initloc string // stack trace of where we were initially created. - - doAllocZero bool - - initialIndexName string - - DeleteEmptyContainer bool - - unlocked int64 - - o Txo - - // NewTx, write operations, Commit and/or Rollback must all take place on - // the same gid and it must the runtime.LockOSThreaded first. Verify - // that we are using the right goroutine in a debug build using the - // gid, stored here, used for NewTx(). - gid uint64 -} - -// sanity check that database is open. -func (tx *BoltTx) sanity() { - if tx.Db.IsClosed() { - panic("cannot operate on closed Bolt") - } -} - -func (tx *BoltTx) Group() *TxGroup { - return tx.o.Group -} - -func (tx *BoltTx) Type() string { - return BoltTxn -} - -func (tx *BoltTx) UseRowCache() bool { - return storage.EnableRowCache() -} - -// Pointer gives us a memory address for the underlying transaction for debugging. -// It is public because we use it in roaring to report invalid container memory access -// outside of a transaction. -func (tx *BoltTx) Pointer() string { - return fmt.Sprintf("%p", tx) -} - -// Rollback rolls back the transaction. -func (tx *BoltTx) Rollback() { - notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) - if !notDone { - return - } - ////vv("bolt rollback tx _sn_ %v; stack \n%v", tx.sn) // , stack()) - if isDebugRun { - tx.sanity() - tx.Db.muDb.Lock() - delete(tx.Db.openTx, tx) - tx.Db.muDb.Unlock() - } - - tx.mu.Lock() - defer tx.mu.Unlock() - - //tx.debugOnlyGidcheck() - _ = tx.tx.Rollback() // must hold tx.mu mutex lock - - tx.o.dbs.Cleanup(tx) -} - -// Commit commits the transaction to permanent storage. -// Commits can handle up to 100k updates to fragments -// at once, but not more. This is a BoltDB imposed limit. -func (tx *BoltTx) Commit() error { - notDone := atomic.CompareAndSwapInt64(&tx.unlocked, 0, 1) - if !notDone { - ////vv("Commit already done") - return nil - } - ////vv("bolt commit tx _sn_ %v; path = '%v'; stack \n%v", tx.sn, tx.Db.path, stack()) - //DumpAllBolt() - - if isDebugRun { - tx.sanity() - tx.Db.muDb.Lock() - delete(tx.Db.openTx, tx) - tx.Db.muDb.Unlock() - } - tx.mu.Lock() - defer tx.mu.Unlock() - - err := tx.tx.Commit() - PanicOn(err) - - tx.o.dbs.Cleanup(tx) - return err -} - -// Readonly returns true iff the BoltTx is read-only. -func (tx *BoltTx) Readonly() bool { - return !tx.write -} - -// RoaringBitmap returns the roaring.Bitmap for all bits in the fragment. -func (tx *BoltTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { - - return tx.OffsetRange(index, field, view, shard, 0, 0, LeftShifted16MaxContainerKey) -} - -// Container returns the requested roaring.Container, selected by fragment and ckey -func (tx *BoltTx) Container(index, field, view string, shard uint64, ckey uint64) (c *roaring.Container, err error) { - - // values returned from Get() are only valid while the transaction - // is open. If you need to use a value outside of the transaction then - // you must use copy() to copy it to another byte slice. - // BUT here we are already inside the Txn. - - bkey := txkey.Key(index, field, view, shard, ckey) - tx.mu.Lock() - - //tx.debugOnlyGidcheck() - - bkt := tx.tx.Bucket(bucketCT) - v := bkt.Get(bkey) - tx.mu.Unlock() - - if v == nil { - // not found - return nil, nil - } - n := len(v) - if n > 0 { - c = tx.toContainer(v[n-1], v[0:(n-1)]) - } - return -} - -var bucketCT = []byte("ct") - -// PutContainer stores rc under the specified fragment and container ckey. -func (tx *BoltTx) PutContainer(index, field, view string, shard uint64, ckey uint64, rc *roaring.Container) error { - - bkey := txkey.Key(index, field, view, shard, ckey) - var by []byte - - ct := roaring.ContainerType(rc) - - switch ct { - case roaring.ContainerArray: - by = fromArray16(roaring.AsArray(rc)) - case roaring.ContainerBitmap: - by = fromArray64(roaring.AsBitmap(rc)) - case roaring.ContainerRun: - by = fromInterval16(roaring.AsRuns(rc)) - case roaring.ContainerNil: - panic("wat? nil container is unexpected, no?!?") - default: - panic(fmt.Sprintf("unknown container type: %v", ct)) - } - tx.mu.Lock() - - bkt := tx.tx.Bucket(bucketCT) - err := bkt.Put(bkey, append(by, ct)) - ////vv("err on put bkey = '%v' was %v", string(bkey), err) - tx.mu.Unlock() - - return err -} - -// RemoveContainer deletes the container specified by the shard and container key ckey -func (tx *BoltTx) RemoveContainer(index, field, view string, shard uint64, ckey uint64) error { - bkey := txkey.Key(index, field, view, shard, ckey) - tx.mu.Lock() - //tx.debugOnlyGidcheck() - - bkt := tx.tx.Bucket(bucketCT) - err := bkt.Delete(bkey) - - tx.mu.Unlock() - return err -} - -// Add sets all the a bits hot in the specified fragment. -func (tx *BoltTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - return tx.addOrRemove(index, field, view, shard, false, a...) -} - -// Remove clears all the specified a bits in the chosen fragment. -func (tx *BoltTx) Remove(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { - return tx.addOrRemove(index, field, view, shard, true, a...) -} - -func (tx *BoltTx) addOrRemove(index, field, view string, shard uint64, remove bool, a ...uint64) (changeCount int, err error) { - if len(a) == 0 { - return 0, nil - } - - // have to sort, b/c input is not always sorted. - sort.Slice(a, func(i, j int) bool { return a[i] < a[j] }) - - var lastHi uint64 = math.MaxUint64 // highbits is always less than this starter. - var rc *roaring.Container - var hi uint64 - var lo uint16 - - for i, v := range a { - - hi, lo = highbits(v), lowbits(v) - if hi != lastHi { - // either first time through, or changed to a different container. - // do we need put the last updated container now? - if i > 0 { - // not first time through, write what we got. - if remove && (rc == nil || rc.N() == 0) { - err = tx.RemoveContainer(index, field, view, shard, lastHi) - PanicOn(err) - } else { - err = tx.PutContainer(index, field, view, shard, lastHi, rc) - PanicOn(err) - } - } - // get the next container - rc, err = tx.Container(index, field, view, shard, hi) - PanicOn(err) - } // else same container, keep adding bits to rct. - chng := false - // rc can be nil before, and nil after, in both Remove/Add below. - // The roaring container add() and remove() methods handle this. - if remove { - rc, chng = rc.Remove(lo) - } else { - rc, chng = rc.Add(lo) - } - if chng { - changeCount++ - } - lastHi = hi - } - // write the last updates. - if remove { - if rc == nil || rc.N() == 0 { - err = tx.RemoveContainer(index, field, view, shard, hi) - PanicOn(err) - } else { - err = tx.PutContainer(index, field, view, shard, hi, rc) - PanicOn(err) - } - } else { - if rc == nil || rc.N() == 0 { - panic("there should be no way to have an empty bitmap AFTER an Add() operation") - } - err = tx.PutContainer(index, field, view, shard, hi, rc) - PanicOn(err) - } - return -} - -// Contains returns exists true iff the bit chosen by key is -// hot (set to 1) in specified fragment. -func (tx *BoltTx) Contains(index, field, view string, shard uint64, key uint64) (exists bool, err error) { - - lo, hi := lowbits(key), highbits(key) - bkey := txkey.Key(index, field, view, shard, hi) - tx.mu.Lock() - - bkt := tx.tx.Bucket(bucketCT) - v := bkt.Get(bkey) - - tx.mu.Unlock() - if v == nil { - return false, nil - } - n := len(v) - if n > 0 { - c := tx.toContainer(v[n-1], v[0:(n-1)]) - exists = c.Contains(lo) - } - return exists, err -} - -// key is the container key for the first roaring Container -// roaring docs: Iterator returns a ContainterIterator which *after* a call to Next(), a call to Value() will -// return the first container at or after key. found will be true if a -// container is found at key. -// -// BoltTx notes: We auto-stop at the end of this shard, not going beyond. -func (tx *BoltTx) ContainerIterator(index, field, view string, shard uint64, firstRoaringContainerKey uint64) (citer roaring.ContainerIterator, found bool, err error) { - - // needle example: "idx:'i';fld:'f';vw:'v';shd:'00000000000000000000';key@00000000000000000000" - needle := txkey.Key(index, field, view, shard, firstRoaringContainerKey) - - // prefix example: "idx:'i';fld:'f';vw:'v';shard:'00000000000000000000';key@" - prefix := txkey.Prefix(index, field, view, shard) - - bi := NewBoltIterator(tx, prefix) - ok := bi.Seek(needle) - if !ok { - return bi, false, nil - } - - // have to compare b/c bolt might give us valid iterator - // that is past our needle if needle isn't present. - return bi, bytes.Equal(bi.lastKey, needle), nil -} - -func (tx *BoltTx) GetFieldSizeBytes(index, field string) (uint64, error) { - return 0, nil -} - -// BoltIterator is the iterator returned from a BoltTx.ContainerIterator() call. -// It implements the roaring.ContainerIterator interface. -type BoltIterator struct { - tx *BoltTx - cur *bolt.Cursor - - prefix []byte - seekto []byte - - // seen counts how many Next() calls we have seen. - // It is used to match roaring.ContainerIterator semantics. - // Also useful for testing. - seen int - - lastKey []byte - lastVal []byte // *roaring.Container - lastOK bool - lastConsumed bool -} - -// NewBoltIterator creates an iterator on tx that will -// only return boltKeys that start with prefix. -func NewBoltIterator(tx *BoltTx, prefix []byte) (bi *BoltIterator) { - - tx.mu.Lock() - - bkt := tx.tx.Bucket(bucketCT) - cur := bkt.Cursor() - tx.mu.Unlock() - - bi = &BoltIterator{ - tx: tx, - cur: cur, - prefix: prefix, - } - - return -} - -// Close tells the database and transaction that the user is done -// with the iterator. -func (bi *BoltIterator) Close() { - // no-op -} - -// Valid returns false if there are no more values in the iterator's range. -func (bi *BoltIterator) Valid() bool { - return bi.lastOK -} - -// Seek allows the iterator to start at needle instead of the global begining. -func (bi *BoltIterator) Seek(needle []byte) (ok bool) { - bi.tx.mu.Lock() - defer bi.tx.mu.Unlock() - - bi.seen++ // if ommited, red TestBolt_ContainerIterator_empty_iteration_loop() - - k, v := bi.cur.Seek(needle) - ////vv("seek to needle '%v' gives key '%v'", string(needle), txkey.ToString(k)) - - if len(k) == 0 { - // not found, no keys after needle. - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - if len(bi.prefix) > 0 { - ok = bytes.HasPrefix(k, bi.prefix) - if !ok { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - } - - bi.lastKey = k - bi.lastVal = v - bi.lastOK = true - bi.lastConsumed = false - - return true -} - -func (bi *BoltIterator) ValidForPrefix(prefix []byte) bool { - if !bi.lastOK { - return false - } - if len(bi.prefix) == 0 { - return true - } - return bytes.HasPrefix(bi.lastKey, bi.prefix) -} - -func (bi *BoltIterator) String() (r string) { - return fmt.Sprintf("BoltIterator{prefix: '%v', seekto: '%v', seen:%v, lastKey:'%v', lastOK:%v, lastConsumed:%v}", string(bi.prefix), string(bi.seekto), bi.seen, string(bi.lastKey), bi.lastOK, bi.lastConsumed) -} - -// Next advances the iterator. -func (bi *BoltIterator) Next() (ok bool) { - //vv("Next; bi.prefix='%v'", string(bi.prefix)) - if bi.lastOK && !bi.lastConsumed { - //vv("lastOk and not consumed, returning wo doing anything") - bi.seen++ - bi.lastConsumed = true - if len(bi.lastVal) == 0 { - panic("bi.lastVal should not have len 0 if lastOK true") - } - return true - } - - var k, v []byte - if bi.seen == 0 { - if len(bi.prefix) == 0 { - bi.tx.mu.Lock() - k, v = bi.cur.First() - bi.tx.mu.Unlock() - } else { - found := bi.Seek(bi.prefix) - // increments bi.seen for us. - if !found { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - // ready to go - return true - } - } - - bi.seen++ -skipEmpty: - if bi.seen > 1 { - bi.tx.mu.Lock() - k, v = bi.cur.Next() - bi.tx.mu.Unlock() - } - - if len(k) == 0 { - // no more - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - if len(bi.prefix) > 0 { - ok = bytes.HasPrefix(k, bi.prefix) - if !ok { - bi.lastKey = nil - bi.lastVal = nil - bi.lastOK = false - bi.lastConsumed = false - return false - } - } - bi.lastKey = k - bi.lastVal = v - if len(v) == 0 { - // actually under !tx.DeleteEmptyContainer, we can have empty containers! - goto skipEmpty - } - bi.lastOK = true - bi.lastConsumed = true - - return true -} - -// Value retrieves what is pointed at currently by the iterator. -func (bi *BoltIterator) Value() (containerKey uint64, c *roaring.Container) { - if !bi.lastOK { - panic("bi.cur not valid") - } - containerKey = txkey.KeyExtractContainerKey(bi.lastKey) - - v := bi.lastVal - n := len(v) - if n > 0 { - c = bi.tx.toContainer(v[n-1], v[0:(n-1)]) - } else { - panic("v should not be empty!") - } - return -} - -// boltFinder implements roaring.IteratorFinder. -// It is used by BoltTx.ForEach() -type boltFinder struct { - tx *BoltTx - index string - field string - view string - shard uint64 - needClose []Closer -} - -// FindIterator lets boltFinder implement the roaring.FindIterator interface. -func (bf *boltFinder) FindIterator(seek uint64) (roaring.ContainerIterator, bool) { - a, found, err := bf.tx.ContainerIterator(bf.index, bf.field, bf.view, bf.shard, seek) - PanicOn(err) - bf.needClose = append(bf.needClose, a) - return a, found -} - -// Close closes all bf.needClose listed Closers. -func (bf *boltFinder) Close() { - for _, i := range bf.needClose { - i.Close() - } -} - -// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE -// the transaction Commits or Rollsback. -func (tx *BoltTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { - - bf := &boltFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} - itr := roaring.NewIterator(bf) - return itr -} - -// ForEach applies fn to each bitmap in the fragment. -func (tx *BoltTx) ForEach(index, field, view string, shard uint64, fn func(i uint64) error) error { - - itr := tx.NewTxIterator(index, field, view, shard) - defer itr.Close() - - // Seek can create many container iterators, thus bf.Close() needClose list. - itr.Seek(0) - // v is the bit we are operating on. - for v, eof := itr.Next(); !eof; v, eof = itr.Next() { - if err := fn(v); err != nil { - return err - } - } - return nil -} - -// ForEachRange applies fn on the selected range of bits on the chosen fragment. -func (tx *BoltTx) ForEachRange(index, field, view string, shard uint64, start, end uint64, fn func(uint64) error) error { - - itr := tx.NewTxIterator(index, field, view, shard) - defer itr.Close() - - itr.Seek(start) - - // v is the bit we are operating on. - for v, eof := itr.Next(); !eof && v < end; v, eof = itr.Next() { - if err := fn(v); err != nil { - return err - } - } - return nil -} - -// Count operates on the full bitmap level, so it sums over all the containers -// in the bitmap. -func (tx *BoltTx) Count(index, field, view string, shard uint64) (uint64, error) { - - a, found, err := tx.ContainerIterator(index, field, view, shard, 0) - PanicOn(err) - defer a.Close() - if !found { - return 0, nil - } - result := int32(0) - for a.Next() { - ckey, cont := a.Value() - _ = ckey - result += cont.N() - } - return uint64(result), nil -} - -// Max is the maximum bit-value in your bitmap. -// Returns zero if the bitmap is empty. Odd, but this is what roaring.Max does. -func (tx *BoltTx) Max(index, field, view string, shard uint64) (uint64, error) { - - prefix := txkey.Prefix(index, field, view, shard) - seekto := txkey.Prefix(index, field, view, shard+1) - - bkt := tx.tx.Bucket(bucketCT) - cur := bkt.Cursor() - - var k, v []byte - k, _ = cur.Seek(seekto) - if k == nil { - // we have nothing >= seekto, but we might have stuff before it, and we'll wrap backwards. - k, v = cur.Prev() - if k == nil { - // empty database - return 0, nil - } - } else { - // we found something >= seekto, so backup by 1. - k, v = cur.Prev() - if k == nil { - // nothing before seekto - return 0, nil - } - } - - // have something, are we in [prefix, seekto) ? - cmp := bytes.Compare(k, prefix) - if cmp >= 0 { - // good, got max in k, v - } else { - return 0, nil // nothing in [prefix, seekto). - } - - n := len(v) - if n == 0 { - return 0, nil - } - - hb := txkey.KeyExtractContainerKey(k) - rc := tx.toContainer(v[n-1], v[0:(n-1)]) - - lb := rc.Max() - return hb<<16 | uint64(lb), nil -} - -// Min returns the smallest bit set in the fragment. If no bit is hot, -// the second return argument is false. -func (tx *BoltTx) Min(index, field, view string, shard uint64) (uint64, bool, error) { - - // Seek can create many container iterators, thus the bf.Close() needClose list. - bf := &boltFinder{tx: tx, index: index, field: field, view: view, shard: shard, needClose: make([]Closer, 0)} - defer bf.Close() - itr := roaring.NewIterator(bf) - - itr.Seek(0) - - // v is the bit we are operating on. - v, eof := itr.Next() - if eof { - return 0, false, nil - } - return v, true, nil -} - -// UnionInPlace unions all the others Bitmaps into a new Bitmap, and then writes it to the -// specified fragment. -func (tx *BoltTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - PanicOn(err) - - rbm.UnionInPlace(others...) - // iterate over the containers that changed within rbm, and write them back to disk. - - it, found := rbm.Containers.Iterator(0) - _ = found // don't care about the value of found, because first containerKey might be > 0 - - for it.Next() { - containerKey, rc := it.Value() - - // TODO: only write the changed ones back, as optimization? - // Compare to ImportRoaringBits. - err := tx.PutContainer(index, field, view, shard, containerKey, rc) - PanicOn(err) - } - return nil -} - -// CountRange returns the count of hot bits in the start, end range on the fragment. -// roaring.countRange counts the number of bits set between [start, end). -func (tx *BoltTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { - - if start >= end { - return 0, nil - } - - skey := highbits(start) - ekey := highbits(end) - - citer, found, err := tx.ContainerIterator(index, field, view, shard, skey) - _ = found - PanicOn(err) - - defer citer.Close() - - // If range is entirely in one container then just count that range. - if skey == ekey { - citer.Next() - _, c := citer.Value() - return uint64(c.CountRange(int32(lowbits(start)), int32(lowbits(end)))), nil - } - - for citer.Next() { - k, c := citer.Value() - if k < skey { - citer.Close() - panic(fmt.Sprintf("should be impossible for k(%v) to be less than skey(%v). tx p=%p", k, skey, tx)) - } - - // k > ekey handles the case when start > end and where start and end - // are in different containers. Same container case is already handled above. - if k > ekey { - break - } - if k == skey { - n += uint64(c.CountRange(int32(lowbits(start)), roaring.MaxContainerVal+1)) - continue - } - if k < ekey { - n += uint64(c.N()) - continue - } - if k == ekey { - n += uint64(c.CountRange(0, int32(lowbits(end)))) - break - } - } - - return n, nil -} - -// OffsetRange creates a new roaring.Bitmap to return in other. For all the -// hot bits in [start, endx) of the chosen fragment, it stores -// them into other but with offset added to their bit position. -// The primary client is doing this, using ShardWidth, already; see -// fragment.rowFromStorage() in fragment.go. For example: -// -// data, err := tx.OffsetRange(f.index, f.field, f.view, f.shard, -// -// f.shard*ShardWidth, rowID*ShardWidth, (rowID+1)*ShardWidth) -// ^ offset ^ start ^ endx -// -// The start and endx arguments are container keys that have been shifted left by 16 bits; -// their highbits() will be taken to determine the actual container keys. This -// is done to conform to the roaring.OffsetRange() argument convention. -// -func (tx *BoltTx) OffsetRange(index, field, view string, shard, offset, start, endx uint64) (other *roaring.Bitmap, err error) { - ////vv("top of BoltTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v)", index, field, view, int(shard), int(offset), int(start), int(endx)) - //defer func() { - ////vv("returning from BoltTx OffsetRange(index='%v', field='%v', view='%v', shard='%v', offset: %v start: %v, end: %v) other returning is: '%#v' stack=\n%v", index, field, view, int(shard), int(offset), int(start), int(endx), asInts(other.Slice()), stack()) - //}() - - // roaring does these three checks in its OffsetRange - if lowbits(offset) != 0 { - panic("offset must not contain low bits") - } - if lowbits(start) != 0 { - panic("range start must not contain low bits") - } - if lowbits(endx) != 0 { - panic("range end must not contain low bits") - } - - other = roaring.NewSliceBitmap() - off := highbits(offset) - hi0, hi1 := highbits(start), highbits(endx) - - needle := txkey.Key(index, field, view, shard, hi0) - prefix := txkey.Prefix(index, field, view, shard) - - it := NewBoltIterator(tx, prefix) - defer it.Close() - it.Seek(needle) - for ; it.ValidForPrefix(prefix); it.Next() { - bkey := it.lastKey - k := txkey.KeyExtractContainerKey(bkey) - - // >= hi1 is correct b/c endx cannot have any lowbits set. - if uint64(k) >= hi1 { - break - } - destCkey := off + (k - hi0) - - v := it.lastVal - n := len(v) - if n == 0 { - continue - } - c := tx.toContainer(v[n-1], v[0:(n-1)]) - other.Containers.Put(destCkey, c.Freeze()) - } - return other, nil -} - -// IncrementOpN increments the tx opcount by changedN -func (tx *BoltTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - tx.opcount += changedN -} - -// ImportRoaringBits handles deletes by setting clear=true. -// rowSet[rowID] returns the number of bit changed on that rowID. -func (tx *BoltTx) ImportRoaringBits(index, field, view string, shard uint64, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { - - n := itr.Len() - if n == 0 { - return - } - rowSet = make(map[uint64]int) - - var currRow uint64 - - var oldC *roaring.Container - for itrKey, synthC := itr.NextContainer(); synthC != nil; itrKey, synthC = itr.NextContainer() { - if rowSize != 0 { - currRow = itrKey / rowSize - } - nsynth := int(synthC.N()) - if nsynth == 0 { - continue - } - // INVAR: nsynth > 0 - - oldC, err = tx.Container(index, field, view, shard, itrKey) - PanicOn(err) - if err != nil { - return - } - - if oldC == nil || oldC.N() == 0 { - // no container at the itrKey in bolt (or all zero container). - if clear { - // changed of 0 and empty rowSet is perfect, no need to change the defaults. - continue - } else { - - changed += nsynth - rowSet[currRow] += nsynth - - err = tx.PutContainer(index, field, view, shard, itrKey, synthC) - if err != nil { - return - } - continue - } - } - - if clear { - existN := oldC.N() // number of bits set in the old container - newC := oldC.Difference(synthC) - - // update rowSet and changes - if newC.N() == existN { - // INVAR: do changed need adjusting? nope. same bit count, - // so no change could have happened. - continue - } else { - changes := int(existN - newC.N()) - changed += changes - rowSet[currRow] -= changes - - if tx.DeleteEmptyContainer && newC.N() == 0 { - err = tx.RemoveContainer(index, field, view, shard, itrKey) - if err != nil { - return - } - continue - } - err = tx.PutContainer(index, field, view, shard, itrKey, newC) - if err != nil { - return - } - continue - } - } else { - // setting bits - - existN := oldC.N() - if existN == roaring.MaxContainerVal+1 { - // completely full container already, set will do nothing. so changed of 0 default is perfect. - continue - } - if existN == 0 { - // can nsynth be zero? No, because of the continue/invariant above where nsynth > 0 - changed += nsynth - rowSet[currRow] += nsynth - err = tx.PutContainer(index, field, view, shard, itrKey, synthC) - if err != nil { - return - } - continue - } - - newC := roaring.Union(oldC, synthC) // UnionInPlace was giving us crashes on overly large containers. - - if roaring.ContainerType(newC) == roaring.ContainerBitmap { - newC.Repair() // update the bit-count so .n is valid. b/c UnionInPlace doesn't update it. - } - if newC.N() != existN { - changes := int(newC.N() - existN) - changed += changes - rowSet[currRow] += changes - - err = tx.PutContainer(index, field, view, shard, itrKey, newC) - if err != nil { - PanicOn(err) - return - } - continue - } - } - } - return -} - -func (tx *BoltTx) toContainer(typ byte, v []byte) (r *roaring.Container) { - - //tx.debugOnlyGidcheck() - - if len(v) == 0 { - return nil - } - - var w []byte - useRowCache := tx.UseRowCache() - if tx.doAllocZero || useRowCache { - // Do electric fence-inspired bad-memory read detection. - // - // The v []byte lives in BoltDB's memory-mapped vlog-file, - // and Bolt will recycle it after tx ends with rollback or commit. - // - // Problem is, at least some operations were not respecting transaction boundaries. - // This technique helped us find them. The rowCache was an example. - // - // See the global const DetectMemAccessPastTx - // at the top of txfactory.go to activate/deactivate this. - // - // Seebs suggested this nice variation: we could use individual mmaps for these - // copies, which would be unusable in production, but workable for testing, and then unmap them, - // which would get us probable segfaults on future accesses to them. - // - // The go runtime also has an -efence flag which may be similarly useful if really pressed. - // - w = make([]byte, len(v)) - copy(w, v) - } else { - w = v - } - return ToContainer(typ, w) -} - -func ToContainer(typ byte, w []byte) (c *roaring.Container) { - switch typ { - case roaring.ContainerArray: - c = roaring.NewContainerArray(toArray16(w)) - case roaring.ContainerBitmap: - c = roaring.NewContainerBitmap(-1, toArray64(w)) - case roaring.ContainerRun: - c = roaring.NewContainerRun(toInterval16(w)) - default: - panic(fmt.Sprintf("unknown container: %v", typ)) - } - c.SetMapped(true) - return c -} - -// StringifiedBoltKeys returns a string with all the container -// keys available in bolt. -func (w *BoltWrapper) StringifiedBoltKeys(optionalUseThisTx Tx, short bool) (r string) { - if optionalUseThisTx == nil { - tx, _ := w.NewTx(!writable, "", Txo{}) - defer tx.Rollback() - r = stringifiedBoltKeysTx(tx.(*BoltTx), short) - return - } - - btx, ok := optionalUseThisTx.(*BoltTx) - if !ok { - return fmt.Sprintf("", optionalUseThisTx) - } - r = stringifiedBoltKeysTx(btx, short) - return -} - -// countBitsSet returns the number of bits set (or "hot") in -// the roaring container value found by the txkey.Key() -// formatted bkey. -func (tx *BoltTx) countBitsSet(bkey []byte) (n int) { - - //tx.debugOnlyGidcheck() - - bkt := tx.tx.Bucket(bucketCT) - v := bkt.Get(bkey) - - if v == nil { - // some queries bkey may not be present! don't panic. - return 0 - } - - n = len(v) - if n > 0 { - rc := tx.toContainer(v[n-1], v[0:(n-1)]) - n = int(rc.N()) - } - return -} - -func (tx *BoltTx) Dump(short bool, shard uint64) { - fmt.Printf("%v\n", stringifiedBoltKeysTx(tx, short)) -} - -func (tx *BoltTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []short_txkey.FieldView, err error) { - bkt := tx.tx.Bucket(bucketCT) - err = bkt.ForEach(func(bkey, v []byte) error { - fv := txkey.FieldViewFromFullKey(bkey) - var shortFV short_txkey.FieldView - shortFV.Field = fv.Field - shortFV.View = fv.View - fvs = append(fvs, shortFV) - return nil - }) - return -} - -// stringifiedBoltKeysTx reports all the bolt keys and a -// corresponding blake3 hash viewable by txn within the entire -// bolt database. -// It also reports how many bits are hot in the roaring container -// (how many bits are set, or 1 rather than 0). -// -// By convention, we must return the empty string if there -// are no keys present. The tests use this to confirm -// an empty database. -func stringifiedBoltKeysTx(tx *BoltTx, short bool) (r string) { - - r = "allkeys:[\n" - it := NewBoltIterator(tx, nil) - defer it.Close() - any := false - for it.Next() { - any = true - - bkey := it.lastKey - key := txkey.ToString(bkey) - ckey := txkey.KeyExtractContainerKey(bkey) - h := "" - srbm := "" - v := it.lastVal - n := len(v) - if n == 0 { - panic("should not have empty v here") - } - h = hash.Blake3sum16(v[0:(n - 1)]) - ct := tx.toContainer(v[n-1], v[0:(n-1)]) - cts := roaring.NewSliceContainers() - cts.Put(ckey, ct) - rbm := &roaring.Bitmap{Containers: cts} - srbm = BitmapAsString(rbm) - - r += fmt.Sprintf("%v -> %v (%v hot)\n", key, h, tx.countBitsSet(bkey)) - if !short { - r += " ......." + srbm + "\n" - } - } - r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) - - if !any { - return "" - } - return "bolt-" + r -} - -func (w *BoltWrapper) DeleteDBPath(dbs *DBShard) (err error) { - path := dbs.pathForType(boltTxn) - err = os.RemoveAll(path) - if err != nil { - return errors.Wrap(err, "DeleteDBPath") - } - return -} - -func (w *BoltWrapper) DeleteField(index, field, fieldPath string) (err error) { - - // TODO(jea) cleanup: I think this fieldPath delete just goes away now. - // remove this commented stuff once we are sure. - // - // under blue-green roaring_bolt, the directory will not be found, b/c roaring will have - // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: - // "If the path does not exist, RemoveAll returns nil (no error)" - - err = os.RemoveAll(fieldPath) - if err != nil { - return errors.Wrap(err, "removing directory") - } - prefix := txkey.FieldPrefix(index, field) - return w.DeletePrefix(prefix) -} - -func (w *BoltWrapper) DeleteFragment(index, field, view string, shard uint64, frag interface{}) error { - prefix := txkey.Prefix(index, field, view, shard) - return w.DeletePrefix(prefix) -} - -func (w *BoltWrapper) DeletePrefix(prefix []byte) error { - - tx, _ := w.NewTx(writable, w.name, Txo{}) - - // NewTx will grab these, so don't lock until after it. - w.muDb.Lock() - - bi := NewBoltIterator(tx.(*BoltTx), prefix) - - for bi.Next() { - //vv("deleting next in cur") - err := bi.cur.Delete() - if err != nil { - w.muDb.Unlock() - panic(err) - } - } - bi.Close() - - // Commit will grab the w.muDb lock, so we must release it first. - w.muDb.Unlock() - - err := tx.Commit() - PanicOn(err) - - return nil -} - -func (tx *BoltTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") - } - var buf bytes.Buffer - sz, err = rbm.WriteTo(&buf) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") - } - return ioutil.NopCloser(&buf), sz, err -} - -func (tx *BoltTx) Options() Txo { - return tx.o -} - -// Sn retreives the serial number of the Tx. -func (tx *BoltTx) Sn() int64 { - return tx.sn -} - -func (c *BoltTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { - return GenericApplyFilter(c, index, field, view, shard, ckey, filter) -} diff --git a/bolt_test.go b/bolt_test.go deleted file mode 100644 index 1dc03b7d0..000000000 --- a/bolt_test.go +++ /dev/null @@ -1,1270 +0,0 @@ -// Copyright 2020 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "fmt" - "math" - "os" - "testing" - - "github.com/molecula/featurebase/v2/roaring" - "github.com/molecula/featurebase/v2/storage" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck -) - -// helpers, each runs their own new txn, and commits if a change/delete -// was made. The txn is rolled back if it is just viewing the data. - -func BoltMustHaveBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, bitvalue uint64) { - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue '%v' was NOT SET!!!", bitvalue)) - } - - tx.Rollback() -} - -func BoltMustNotHaveBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, bitvalue uint64) { - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if exists { - panic(fmt.Sprintf("ARG bitvalue '%v' WAS SET but should not have been.!!!", bitvalue)) - } - tx.Rollback() -} - -func BoltMustSetBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - // add a bit - changed, err := tx.Add(index, field, view, shard, putme) - if changed != 1 { - panic("should have 1 bit changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic("ARG putme was NOT SET!!!") - } - PanicOn(tx.Commit()) -} - -func BoltMustDeleteBitvalueContainer(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - hi := highbits(putme) - PanicOn(tx.RemoveContainer(index, field, view, shard, hi)) - PanicOn(tx.Commit()) -} - -func BoltMustDeleteBitvalue(dbwrap *BoltWrapper, index, field, view string, shard uint64, putme uint64) { - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - _, err := tx.Remove(index, field, view, shard, putme) - PanicOn(err) - PanicOn(tx.Commit()) -} - -func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) { - var err error - fn := path - PanicOn(os.RemoveAll(fn)) - ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, &storage.Config{FsyncEnabled: false}) - PanicOn(err) - w = ww.(*BoltWrapper) - - // verify it is empty - allkeys := w.StringifiedBoltKeys(nil, false) - if allkeys != "" { - panic(fmt.Sprintf("freshly created database was not empty! had keys:'%v'", allkeys)) - } - - return w, func() { - w.Close() - PanicOn(os.RemoveAll(fn)) - } -} - -// end of helper utilities -////////////////////////// - -////////////////////////// -// begin Tx method tests - -func TestBolt_DeleteFragment(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_DeleteFragment") - defer clean() - defer dbwrap.Close() - index, field, shard := "i", "f", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} - views := []string{"v1", "v2"} - for _, view := range views { - for _, v := range bits { - changed, err := tx.Add(index, field, view, shard, v) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - } - } - - for _, view := range views { - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - } - } - err := tx.Commit() - PanicOn(err) - - // end of setup - - victim := "v1" - survivor := "v2" - err = dbwrap.DeleteFragment(index, field, victim, shard, nil) - PanicOn(err) - - tx, _ = dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - for _, view := range views { - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if view == survivor { - if !exists { - panic(fmt.Sprintf("ARG survivor died : bit %v", v)) - } - } else if view == victim { // victim, should have been deleted - if exists { - panic(fmt.Sprintf("ARG victim lived : bit %v", v)) - } - } - } - } -} - -func TestBolt_Max_on_many_containers(t *testing.T) { - path := "TestBolt_Max_on_many_containers" - dbwrap, clean := mustOpenEmptyBoltWrapper(path) - - defer clean() - defer dbwrap.Close() - index, field, view := "i", "f", "v" - - // 099 - // 101 - // 199 - // 300 - // 399 - // - // find max in [300,400) and get 399 - // find max in [000,100) and get 099 - // find max in [100,200) and get 199 - // find max in [400,500) and get nothing back - // find max in [200,300) and get nothing back - - shards := []int{99, 101, 199, 300, 399} - - for _, sh := range shards { - shard := uint64(sh) - for _, pm := range shards { - putme := uint64(pm) - if putme > shard { - continue - } - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - } - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - for _, shard := range shards { - max, err := tx.Max(index, field, view, uint64(shard)) - PanicOn(err) - if max != uint64(shard) { - panic(fmt.Sprintf("expected max (%v) to be == shard = %v", max, shard)) - } - } - - // check for not found - max, err := tx.Max(index, field, view, uint64(200)) - PanicOn(err) - if max != 0 { - panic("expected not found to give 0 max back with nil err") - } - max, err = tx.Max(index, field, view, uint64(400)) - PanicOn(err) - if max != 0 { - panic("expected not found to give 0 max back with nil err") - } - -} - -// and the rest - -func TestBolt_SetBitmap(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_SetBitmap") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - bitvalue := uint64(0) - changed, err := tx.Add(index, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - - err = tx.Commit() - PanicOn(err) - - // - // commited, so should be visible outside the txn - // - - tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) - exists, err = tx2.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!! on tx2") - } - - n, err := tx2.Count(index, field, view, shard) - PanicOn(err) - if n != 1 { - panic(fmt.Sprintf("should have Count 1; instead n = %v", n)) - } - tx2.Rollback() -} - -func TestBolt_OffsetRange(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_OffsetRange") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - bitvalue := uint64(1 << 20) - changed, err := tx.Add(index, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - bitvalue2 := uint64(1<<20 + 1) - changed, err = tx.Add(index, field, view, shard, bitvalue2) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - exists, err = tx.Contains(index, field, view, shard, bitvalue2) - PanicOn(err) - if !exists { - panic("ARG bitvalue2 was NOT SET!!!") - } - - err = tx.Commit() - PanicOn(err) - - offset := uint64(0 << 20) - start := uint64(0 << 16) - endx := bitvalue + 1<<16 - - tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) - rbm2, err := tx2.OffsetRange(index, field, view, shard, offset, start, endx) - PanicOn(err) - tx2.Rollback() - - // should see our 1M value - s2 := BitmapAsString(rbm2) - expect2 := "c(1048576, 1048577)" - if s2 != expect2 { - panic(fmt.Sprintf("s2='%v', but expected '%v'", s2, expect2)) - } - - // now offset by 2M - offset = uint64(2 << 20) - tx3, _ := dbwrap.NewTx(!writable, index, Txo{}) - rbm3, err := tx3.OffsetRange(index, field, view, shard, offset, start, endx) - PanicOn(err) - tx3.Rollback() - - //expect to see 3M == 3145728 - s3 := BitmapAsString(rbm3) - expect3 := "c(3145728, 3145729)" - - if s3 != expect3 { - panic(fmt.Sprintf("s3='%v', but expected '%v'", s3, expect3)) - } -} - -func TestBolt_Count_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Count_on_many_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{0, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - n, err := tx.Count(index, field, view, shard) - PanicOn(err) - if int(n) != len(putmeValues) { - panic(fmt.Sprintf("expected Count of %v but got n=%v", len(putmeValues), n)) - } -} - -func TestBolt_Count_dense_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Count_dense_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - - expected := 0 - for i := uint64(0); i < (1<<16)+2; i += 2 { - changed, err := tx.Add(index, field, view, shard, i) - PanicOn(err) - if changed <= 0 { - panic("wat? should have changed") - } - expected++ - } - defer tx.Rollback() - - n, err := tx.Count(index, field, view, shard) - PanicOn(err) - if int(n) != expected { - panic(fmt.Sprintf("expected Count of %v but got n=%v", expected, n)) - } -} - -func TestBolt_ContainerIterator_on_empty(t *testing.T) { - // iterate on empty container, should not find anything. - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - bitvalue := uint64(0) - citer, found, err := tx.ContainerIterator(index, field, view, shard, bitvalue) - PanicOn(err) - defer citer.Close() - if found { - panic("should not have found anything") - } - PanicOn(err) -} - -func TestBolt_ContainerIterator_on_one_bit(t *testing.T) { - // set one bit, iterate. - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_on_one_bit") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - bitvalue := uint64(42) - - // add a bit - changed, err := tx.Add(index, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - - // same Tx, continues in use. - - citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(bitvalue)) - if !found { - panic("ContainerIterator did not find the 42 bit") - } - PanicOn(err) - defer citer.Close() - - loopCount := 0 - for citer.Next() { - key, container := citer.Value() - if key != 0 { - panic("42 should have had key 0") - } - if container == nil { - panic("container was nil") - } - if container.N() != 1 { - panic("put a bit in, but size of container was not 1") - } - if !container.Contains(lowbits(bitvalue)) { - panic("container did not have our bitvalue!") - } - loopCount++ - if loopCount > 0 { // happier linter - break - } - } - if loopCount != 1 { - panic("ContainerIterator did not return a citer that scanned our set bit") - } -} - -func TestBolt_ContainerIterator_on_one_bit_fail_to_find(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_on_one_bit_fail_to_find") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - putme := uint64(1<<16) + 3 // in the key:1 container - searchme := putme + 1 - - // add a bit - changed, err := tx.Add(index, field, view, shard, putme) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic("ARG putme was NOT SET!!!") - } - - // same Tx, continues in use. - - citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(searchme)) - if !found { - panic("ContainerIterator did not find the searchme") - } - defer citer.Close() - loopCount := 0 - for citer.Next() { - key, container := citer.Value() - if key != 1 { - panic("Containeriterator searching for highbits(searchme) should not have had a bit") - } - if container == nil { - panic("container was nil") - } - if container.N() != 1 { - panic("put a bit in, but size of container was not 1") - } - if container.Contains(lowbits(searchme)) { - panic("container should have putme but not our searchme!") - } - loopCount++ - // only want first pass. keep linter happy by avoiding raw break - if loopCount > 0 { - break - } - } - PanicOn(err) -} - -func TestBolt_ContainerIterator_empty_iteration_loop(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ContainerIterator_empty_iteration_loop") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - putme := uint64(1<<16) + 3 // in the key:1 container - searchme := uint64(1 << 17) // in the next container, key:2 - - // add a bit - changed, err := tx.Add(index, field, view, shard, putme) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic("ARG putme was NOT SET!!!") - } - - // same Tx, continues in use. - - citer, found, err := tx.ContainerIterator(index, field, view, shard, highbits(searchme)) - PanicOn(err) - if found { - panic("ContainerIterator found the searchme, when it should not have") - } - defer citer.Close() - if citer.Next() { - panic("expected no looping, 0 iterations, b/c started searchme past our data in putme") - } - - // expect to see a blow up from the citer.Value() call, verify that we do. - func() { - defer func() { - r := recover() - if r == nil { - panic("expected a panic from citer.Value() in this case") - } - }() - citer.Value() // should panic - }() - -} - -func TestBolt_ForEach_on_one_bit(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ForEach_on_one_bit") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - bitvalue := uint64(42) - - // add a bit - changed, err := tx.Add(index, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - exists, err := tx.Contains(index, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - - // same Tx, continues in use. - count := 0 - err = tx.ForEach(index, field, view, shard, func(v uint64) error { - if v != bitvalue { - panic(fmt.Sprintf("bitvalue corrupt got %v want %v", v, bitvalue)) - } - count += 1 - return nil - }) - PanicOn(err) - if count != 1 { - panic(fmt.Sprintf("Expected single iteration got %v ", count)) - } -} - -func TestBolt_RemoveContainer_one_bit_test(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_RemoveContainer_one_bit_test") - defer clean() - defer dbwrap.Close() - - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{0, 13, 77, 1511} - - for _, putme := range putmeValues { - - // a) delete of whole container in a seperate txn. Commit should establish the deletion. - - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // b) deletion + rollback on the txn should restore the deleted bit - - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // delete, but rollback instead of commit - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - hi := highbits(putme) - PanicOn(tx.RemoveContainer(index, field, view, shard, hi)) - tx.Rollback() - - // verify that the rollback undid the deletion. - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // c) within one Tx, after delete it should be gone as viewed within the txn. - tx, _ = dbwrap.NewTx(writable, index, Txo{}) - hi = highbits(putme) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme)) - } - - PanicOn(tx.RemoveContainer(index, field, view, shard, hi)) - - exists, err = tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if exists { - panic(fmt.Sprintf("ARG putme '%v' was SET even after RemoveContiner in this txn.", putme)) - } - - tx.Rollback() - - // verify that the rollback undid the deletion. - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - // leave with clean slate - BoltMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - } -} - -func TestBolt_Remove_one_bit_test(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Remove_one_bit_test") - defer clean() - defer dbwrap.Close() - - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{0, 13, 77, 1511} - - for _, putme := range putmeValues { - - // a) delete of whole container in a seperate txn. Commit should establish the deletion. - - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustDeleteBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // b) deletion + rollback on the txn should restore the deleted bit - - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // delete, but rollback instead of commit - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - hi, lo := highbits(putme), lowbits(putme) - _, _ = hi, lo - _, err := tx.Remove(index, field, view, shard, hi) - PanicOn(err) - tx.Rollback() - - // verify that the rollback undid the deletion. - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - - // c) within one Tx, after delete it should be gone as viewed within the txn. - tx, _ = dbwrap.NewTx(writable, index, Txo{}) - - exists, err := tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG putme '%v' was NOT SET!!!", putme)) - } - - mustRemove(tx.Remove(index, field, view, shard, putme)) - - exists, err = tx.Contains(index, field, view, shard, putme) - PanicOn(err) - if exists { - panic(fmt.Sprintf("ARG putme '%v' was SET even after Remove in this txn.", putme)) - } - - tx.Rollback() - - // verify that the rollback undid the deletion. - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - // leave with clean slate - BoltMustDeleteBitvalueContainer(dbwrap, index, field, view, shard, putme) - } -} - -func TestBolt_Min_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_Min_on_many_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - // verify no containers flag works - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - min, containersExist, err := tx.Min(index, field, view, shard) - _ = min - PanicOn(err) - if containersExist { - panic("no containers should exist") - } - tx.Rollback() - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ = dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - min, containersExist, err = tx.Min(index, field, view, shard) - PanicOn(err) - if !containersExist { - panic("containers should exist") - } - expected := putmeValues[0] - if min != expected { - panic(fmt.Sprintf("expected Min() of %v but got min=%v", expected, min)) - } -} - -func TestBolt_CountRange_on_many_containers(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_CountRange_on_many_containers") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - // verify no containers flag works - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - n, err := tx.CountRange(index, field, view, shard, 0, math.MaxUint64) - PanicOn(err) - if n != 0 { - panic("no containers should exist") - } - tx.Rollback() - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ = dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - n, err = tx.CountRange(index, field, view, shard, 0, math.MaxUint64) - PanicOn(err) - if n == 0 { - panic("containers should exist") - } - expected := uint64(len(putmeValues)) - if n != expected { - panic(fmt.Sprintf("expected CountRange() of %v but got n=%v", expected, n)) - } -} - -func TestBolt_CountRange_middle_container(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_CountRange_middle_container") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - // pick out just the middle container with the 1 bit set on it. - n, err := tx.CountRange(index, field, view, shard, 4, (2<<16)+1) - PanicOn(err) - if n != 1 { - panic("middle 1 bit container should exist") - } -} - -func TestBolt_CountRange_many_middle_container(t *testing.T) { - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_CountRange_many_middle_container") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{3, 2 << 16, 4 << 16} - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - // get them all - n, err := tx.CountRange(index, field, view, shard, 0, (4<<16)+1) - PanicOn(err) - if n != 3 { - panic("count should have been all 3 bits") - } -} - -func TestBolt_UnionInPlace(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_UnionInPlace") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - putmeValues := []uint64{3, 2 << 16} - - others := roaring.NewBitmap() - others2 := roaring.NewBitmap() - others3 := roaring.NewBitmap() - // populate others with putmeValues +1 into others - - for _, putme := range putmeValues { - BoltMustNotHaveBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - BoltMustHaveBitvalue(dbwrap, index, field, view, shard, putme) - } - - tx2, _ := dbwrap.NewTx(!writable, index, Txo{}) - n, err := tx2.Count(index, field, view, shard) - PanicOn(err) - if n != 2 { - panic("should have 2 bits set") - } - tx2.Rollback() - - for _, putme := range putmeValues { - mustAddR(others.Add(putme)) // should not change count, b/c putme already in the rbm - mustAddR(others.Add(putme + 1)) - mustAddR(others2.Add(putme + 2)) - } - mustAddR(others3.Add(4 << 16)) // outside the 2<<16 container - - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - err = tx.UnionInPlace(index, field, view, shard, others, others2, others3) - PanicOn(err) - - // end game, check we got the union. - rbm, err := tx.RoaringBitmap(index, field, view, shard) - PanicOn(err) - n = rbm.Count() - if n != 7 { - panic("should have a total 3 + 3 +1 = 7 bits set on the containers") - } -} - -func TestBolt_RoaringBitmap(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_RoaringBitmap") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - - expected := uint64(3) - putme := expected - BoltMustSetBitvalue(dbwrap, index, field, view, shard, putme) - - tx, _ := dbwrap.NewTx(!writable, index, Txo{}) - defer tx.Rollback() - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - PanicOn(err) - - slc := rbm.Slice() - if slc[0] != uint64(expected) { - panic(fmt.Sprintf("should have gotten %v back", expected)) - } -} - -func TestBolt_ImportRoaringBits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ImportRoaringBits") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - tx.(*BoltTx).DeleteEmptyContainer = true // traditional lmdb Tx behavior, but not Roaring. - - //bitvalue := uint64(42) - - // get some roaring bits, get an itr RoaringIterator from them - rowSize := uint64(0) - //bits := []uint64{0} - bits := []uint64{0, 2, 5, 1<<16 + 1, 2 << 16} - data := getTestBitmapAsRawRoaring(bits...) - itr, err := roaring.NewRoaringIterator(data) - PanicOn(err) - clear := false - logme := false - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != len(bits) { - panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now test the union in place with the same set gives no change. - - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != 0 { - panic(fmt.Sprintf("should have not changed any bits on the second import, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now test the clear path - clear = true - - for _, v := range bits { - // clear 1 bit at a time - data := getTestBitmapAsRawRoaring(v) - itr, err := roaring.NewRoaringIterator(data) - PanicOn(err) - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != 1 { - panic(fmt.Sprintf("should have changed 1 bit: '%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - PanicOn(err) - } - n, err := tx.Count(index, field, view, shard) - PanicOn(err) - if n != 0 { - panic(fmt.Sprintf("n = %v not zero so the clearbits didn't happen!", n)) - } - allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) - - // should have no keys - if allkeys != "" { - panic("bolt should have no keys now") - } -} - -func TestBolt_ImportRoaringBits_set_nonoverlapping_bits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ImportRoaringBits_set_nonoverlapping_bits") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - // get some roaring bits, get an itr RoaringIterator from them - rowSize := uint64(0) - //bits := []uint64{0} - bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} - data := getTestBitmapAsRawRoaring(bits...) - itr, err := roaring.NewRoaringIterator(data) - PanicOn(err) - - bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16} - data2 := getTestBitmapAsRawRoaring(bits2...) - itr2, err := roaring.NewRoaringIterator(data2) - PanicOn(err) - - clear := false - logme := false - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != len(bits) { - panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now import the 2nd, overlapping set and set them. - - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil) - _ = rowSet - if changed != 4 { - panic(fmt.Sprintf("should have changed 2 bits: the 1 and the 3, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - PanicOn(err) -} - -func TestBolt_ImportRoaringBits_clear_nonoverlapping_bits(t *testing.T) { - - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_ImportRoaringBits_clear_nonoverlapping_bits") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - defer tx.Rollback() - - // get some roaring bits, get an itr RoaringIterator from them - rowSize := uint64(0) - //bits := []uint64{0} - bits := []uint64{0, 2, 1 << 16, 1<<16 + 2} //, 5, 1<<16 + 1, 2 << 16} - data := getTestBitmapAsRawRoaring(bits...) - itr, err := roaring.NewRoaringIterator(data) - PanicOn(err) - - bits2 := []uint64{1, 2, 3, 1<<16 + 1, 1<<16 + 2, 1<<16 + 3} //, 5, 1<<16 + 1, 2 << 16} - data2 := getTestBitmapAsRawRoaring(bits2...) - itr2, err := roaring.NewRoaringIterator(data2) - PanicOn(err) - - clear := false - logme := false - - changed, rowSet, err := tx.ImportRoaringBits(index, field, view, shard, itr, clear, logme, rowSize, nil) - _ = rowSet - if changed != len(bits) { - panic(fmt.Sprintf("should have changed %v bits: changed='%v', rowSet='%#v', err='%v'", len(bits), changed, rowSet, err)) - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("ARG bitvalue was NOT SET!!! '%v'", v)) - } - } - - // now import the 2nd overlapping set and clear them. - clear = true - - changed, rowSet, err = tx.ImportRoaringBits(index, field, view, shard, itr2, clear, logme, rowSize, nil) - _ = rowSet - if changed != 2 { - panic(fmt.Sprintf("should have changed 1 bit: the 2, but we see changed='%v', rowSet='%#v', err='%v'", changed, rowSet, err)) - } - PanicOn(err) - - n, err := tx.Count(index, field, view, shard) - PanicOn(err) - if n != 2 { // just the 0 and the 1<<16 bits should be left set. - panic(fmt.Sprintf("n = %v not 2 so the clearbits didn't happen!", n)) - } - -} - -func TestBolt_DeleteIndex(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_DeleteIndex") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - bitvalue := uint64(777) - bits := []uint64{0, 3, 1 << 16, 1<<16 + 3, 8 << 16} - for _, v := range bits { - changed, err := tx.Add(index, field, view, shard, v) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - } - - index2 := "i2" // should not be deleted, even though it shares a prefix with 'i' - changed, err := tx.Add(index2, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - - for _, v := range bits { - exists, err := tx.Contains(index, field, view, shard, v) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!!") - } - } - exists, err := tx.Contains(index2, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic("ARG bitvalue was NOT SET!!! on index2") - } - err = tx.Commit() - PanicOn(err) - - // end of setup - err = dbwrap.DeleteIndex(index) - PanicOn(err) - - tx, _ = dbwrap.NewTx(!writable, index2, Txo{}) - defer tx.Rollback() - exists, err = tx.Contains(index2, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2)) - } - - for _, v := range bits { - exists, err = tx.Contains(index, field, view, shard, v) - PanicOn(err) - if exists { - allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) - panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) - } - } -} - -func TestBolt_DeleteIndex_over100k(t *testing.T) { - - // setup - dbwrap, clean := mustOpenEmptyBoltWrapper("TestBolt_DeleteIndex_over100k") - defer clean() - defer dbwrap.Close() - index, field, view, shard := "i", "f", "v", uint64(0) - tx, _ := dbwrap.NewTx(writable, index, Txo{}) - bitvalue := uint64(777) - limit := uint64(100002) // default batch size in DeleteIndex is 100k keys per delete transaction. - //limit := uint64(101) - for v := uint64(1); v < limit; v++ { - // shift by << 16 to get into a different shard - changed, err := tx.Add(index, field, view, shard, v<<16) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - if v%100000 == 0 { - PanicOn(tx.Commit()) - tx, _ = dbwrap.NewTx(writable, index, Txo{}) - } - } - - index2 := "i2" // should not be deleted, even though it shares a prefix with 'i' - changed, err := tx.Add(index2, field, view, shard, bitvalue) - if changed <= 0 { - panic("should have changed") - } - PanicOn(err) - err = tx.Commit() - PanicOn(err) - - // end of setup - err = dbwrap.DeleteIndex(index) - PanicOn(err) - - tx, _ = dbwrap.NewTx(!writable, index2, Txo{}) - defer tx.Rollback() - exists, err := tx.Contains(index2, field, view, shard, bitvalue) - PanicOn(err) - if !exists { - panic(fmt.Sprintf("after delete of '%v', another index '%v' was gone too?!?", index, index2)) - } - - for v := uint64(0); v < limit; v++ { - exists, err = tx.Contains(index, field, view, shard, v<<16) - PanicOn(err) - if exists { - allkeys := stringifiedBoltKeysTx(tx.(*BoltTx), false) - panic(fmt.Sprintf("after delete of index '%v', bit v=%v was not gone?!?; allkeys='%v'", index, v, allkeys)) - } - } -} - -func TestBolt_HasData(t *testing.T) { - - db, clean := mustOpenEmptyBoltWrapper("TestBolt_SliceOfShards") - defer clean() - defer db.Close() - - // HasData should start out false. - hasAnything, err := db.HasData() - if err != nil { - t.Fatal(err) - } - if hasAnything { - t.Fatalf("HasData reported existing data on an empty database") - } - - // check that HasData sees a committed record. - - index, field, view, shard, putme := "i", "f", "v", uint64(123), uint64(42) - BoltMustSetBitvalue(db, index, field, view, shard, putme) - - // HasData(false) should now report data - hasAnything, err = db.HasData() - if err != nil { - t.Fatal(err) - } - if !hasAnything { - t.Fatalf("HasData() reported no data on a database that has bits written to it") - } -} diff --git a/catcher.go b/catcher.go index b807b8ea9..92ce5b2b9 100644 --- a/catcher.go +++ b/catcher.go @@ -15,9 +15,6 @@ package pilosa import ( - "fmt" - "io" - "github.com/molecula/featurebase/v2/roaring" txkey "github.com/molecula/featurebase/v2/short_txkey" . "github.com/molecula/featurebase/v2/vprint" @@ -42,40 +39,18 @@ func init() { var _ Tx = (*catcherTx)(nil) -func (c *catcherTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - c.b.IncrementOpN(index, field, view, shard, changedN) -} - func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { return c.b.NewTxIterator(index, field, view, shard) } -func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { +func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { defer func() { if r := recover(); r != nil { AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack()) PanicOn(r) } }() - return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) -} - -func (c *catcherTx) Dump(short bool, shard uint64) { - c.b.Dump(short, shard) -} - -func (c *catcherTx) Readonly() bool { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Readonly() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.Readonly() -} - -func (tx *catcherTx) Pointer() string { - return fmt.Sprintf("%p", tx) + return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize) } func (c *catcherTx) Rollback() { @@ -143,14 +118,6 @@ func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key return c.b.RemoveContainer(index, field, view, shard, key) } -func (c *catcherTx) UseRowCache() bool { - return c.b.UseRowCache() -} - -func (c *catcherTx) IsDone() bool { - return c.b.IsDone() -} - func (c *catcherTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { defer func() { @@ -250,17 +217,6 @@ func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool, return c.b.Min(index, field, view, shard) } -func (c *catcherTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.UnionInPlace(index, field, view, shard, others...) -} - func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { defer func() { @@ -283,33 +239,10 @@ func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start, return c.b.OffsetRange(index, field, view, shard, offset, start, end) } -func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) -} - func (c *catcherTx) Type() string { return c.b.Type() } -func (c *catcherTx) Group() *TxGroup { - return c.b.Group() -} - -func (c *catcherTx) Options() Txo { - return c.b.Options() -} - -// Sn retreives the serial number of the Tx. -func (c *catcherTx) Sn() int64 { - return c.b.Sn() -} - func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { return GenericApplyFilter(c, index, field, view, shard, ckey, filter) } diff --git a/cluster_internal_test.go b/cluster_internal_test.go index cc55ae9aa..3d204ae92 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -17,7 +17,6 @@ package pilosa import ( "fmt" "math/rand" - "net" "reflect" "strings" "testing" @@ -30,57 +29,8 @@ import ( "github.com/molecula/featurebase/v2/testhook" "github.com/molecula/featurebase/v2/topology" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck - "github.com/pkg/errors" ) -// GlobalPortMap avoids many races and port conflicts when setting -// up ports for test clusters. Used for tests only. -var globalPortMap *GlobalPortMapper - -func init() { - globalPortMap = NewGlobalPortMapper(300) -} - -// GlobalPortMapper maintains a pool of available ports by -// holding them open until GetPort() is called. -type GlobalPortMapper struct { - availPorts map[int]net.Listener -} - -// reserve n ports -func NewGlobalPortMapper(n int) (pm *GlobalPortMapper) { - - pm = &GlobalPortMapper{ - availPorts: make(map[int]net.Listener), - } - for i := 0; i < n; i++ { - lsn, err := net.Listen("tcp", ":0") - if err != nil { - panic(errors.Wrap(err, "trying to listen on ephemeral port")) - } - r := lsn.Addr() - port := r.(*net.TCPAddr).Port - pm.availPorts[port] = lsn - } - return -} - -func (pm *GlobalPortMapper) GetPort() (port int, err error) { - for port, lsn := range pm.availPorts { - lsn.Close() - return port, nil - } - return -1, fmt.Errorf("no more ports available") -} - -func (pm *GlobalPortMapper) MustGetPort() int { - port, err := pm.GetPort() - if err != nil { - panic(err) - } - return port -} - // Ensure that fragCombos creates the correct fragment mapping. func TestFragCombos(t *testing.T) { uri0, err := pnet.NewURIFromAddress("host0") diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index 8b5b62d6c..627d56ef2 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -310,7 +310,7 @@ func Migrate(dataDir, backupPath string) error { return err } key := string(txkey.Prefix(index, field, view, shard)) - _, _, err = tx.ImportRoaringBits(key, itr, clear, log, rowSize, nil) + _, _, err = tx.ImportRoaringBits(key, itr, clear, log, rowSize) if err != nil { tx.Rollback() return err diff --git a/ctl/server.go b/ctl/server.go index 6c4405135..71c26bb3c 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -94,7 +94,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { // over-ride. // TODO: the comment above was carried over from the PILOSA_TXSRC flag, but // we should confirm that this still applies. - flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend)) + flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring or rbf. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend)) flags.BoolVar(&srv.Config.Storage.FsyncEnabled, "storage.fsync", true, "enable fsync fully safe flush-to-disk") // RowcacheOn diff --git a/dbshard.go b/dbshard.go index a86cc4ab7..15bd41915 100644 --- a/dbshard.go +++ b/dbshard.go @@ -57,12 +57,10 @@ type DBIndex struct { type DBWrapper interface { NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) - DeleteDBPath(dbs *DBShard) error Close() error DeleteFragment(index, field, view string, shard uint64, frag interface{}) error DeleteField(index, field, fieldPath string) error OpenListString() string - OpenSnList() (sns []int64) Path() string HasData() (has bool, err error) SetHolder(h *Holder) @@ -82,134 +80,52 @@ type DBShard struct { Shard uint64 Open bool - // With RWMutex, the blue-green Tx can start and commit - // atomically. - mut sync.RWMutex - - types []txtype - stypes []string + typ txtype + styp string hasRoaring bool // if either of the types is roaringTxn - W []DBWrapper + W DBWrapper ParentDBIndex *DBIndex idx *Index per *DBPerShard - useOpenList int - closed bool - - isBlueGreen bool + closed bool } func (dbs *DBShard) DeleteFragment(index, field, view string, shard uint64, frag interface{}) (err error) { - for _, w := range dbs.W { - err = w.DeleteFragment(index, field, view, shard, frag) - if err != nil { - return err - } + if index != dbs.Index { + return fmt.Errorf("DeleteFragment called on DBShard for %q with index %q", dbs.Index, index) } - return + if shard != dbs.Shard { + return fmt.Errorf("DeleteFragment called on DBShard for %d with shard %d", dbs.Shard, shard) + } + return dbs.W.DeleteFragment(index, field, view, shard, frag) } func (dbs *DBShard) DeleteFieldFromStore(index, field, fieldPath string) (err error) { - for _, w := range dbs.W { - err = w.DeleteField(index, field, fieldPath) - if err != nil { - return err - } + if index != dbs.Index { + return fmt.Errorf("DeleteFieldFromStore called on DBShard for %q with index %q", dbs.Index, index) } - return + return dbs.W.DeleteField(index, field, fieldPath) } func (dbs *DBShard) Close() (err error) { - for _, w := range dbs.W { - err = w.Close() - if err != nil { - return err - } - } dbs.closed = true - return -} - -// Cleanup must be called at every commit/rollback of a Tx, in -// order to release the read-write mutex that guarantees a single -// writer at a time. Each tx must take care to call cleanup() -// exactly once. examples: -// tx.o.dbs.Cleanup(tx) -// tx.Options().dbs.Cleanup(tx) -// -func (dbs *DBShard) Cleanup(tx Tx) { - if dbs == nil { - return // some tests are using Tx only, no dbs available. - } - //vv("gid %v top of DBShard %v Cleanup for tx.Sn = %v; dbs=%p; is 2nd: %v; type='%v'; dbs.stypes='%#v'", curGID(), dbs.Shard, tx.Sn(), dbs, tx.Type() == dbs.stypes[1], tx.Type(), dbs.stypes) - if !dbs.hasRoaring { - if dbs.isBlueGreen { - // only release on the 2nd Tx's cleanup - if tx.Type() == dbs.stypes[1] { - if tx.Readonly() { - dbs.mut.RUnlock() - //vv("gid %v released read-lock on shard %v", curGID(), dbs.Shard) - } else { - dbs.mut.Unlock() - //vv("gid %v released write-lock on shard %v", curGID(), dbs.Shard) - } - } - } - } + return dbs.W.Close() } func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) { - - if dbs.isBlueGreen { - // enforce only one writer at a time. The dbs.mut is held until - // the Tx finishes. This makes the two Tx in the blue-green Tx atomic. - if !dbs.hasRoaring { - if write { - //vv("shard %v about to write lock by gid %v; stack =\n%v", dbs.Shard, curGID(), stack()) - dbs.mut.Lock() - //vv("shard %v was write locked by gid %v; stack =\n%v", dbs.Shard, curGID(), stack()) - } else { - //vv("shard %v about to be read locked by gid %v; stack=\n%v", dbs.Shard, curGID(), stack()) - dbs.mut.RLock() - //vv("shard %v was read locked by gid %v; stack=\n%v", dbs.Shard, curGID(), stack()) - } - } + if initialIndexName != dbs.Index { + return nil, fmt.Errorf("NewTx called on DBShard for %q with index %q", dbs.Index, initialIndexName) } if o.dbs != dbs { - PanicOn(fmt.Sprintf("TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs)) + return nil, fmt.Errorf("dbs mismatch: TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs) } if o.Shard != dbs.Shard { - PanicOn(fmt.Sprintf("shard disagreement! o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard))) + return nil, fmt.Errorf("shard disagreement: o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard)) } - var txns []Tx - - for _, w := range dbs.W { - tx, err = w.NewTx(write, initialIndexName, o) - if err != nil { - return nil, err - } - txns = append(txns, tx) - } - if len(txns) == 1 { - return - } - // blue green - tx, err = dbs.per.txf.newBlueGreenTx(txns[0], txns[1], o.Index, o), nil - //vv("dbshard returning blue-green tx sn %v", tx.Sn()) - return -} - -func (dbs *DBShard) DeleteDBPath() (err error) { - for _, w := range dbs.W { - err = w.DeleteDBPath(dbs) - if err != nil { - return err - } - } - return + return dbs.W.NewTx(write, initialIndexName, o) } type flatkey struct { @@ -228,34 +144,26 @@ type DBPerShard struct { // Easily see how many we have. Flatmap map[flatkey]*DBShard - types []txtype + typ txtype hasRoaring bool txf *TxFactory holder *Holder - // which of our types is not-roaring, since - // roaring doesn't keep a list of open Tx sn. - // or default to the 2nd. - useOpenList int - // cache the shards per index to avoid excessive - // directory scans of the index directory. Keep per - // txtype to allow blue-green migrate open to be fast too. + // directory scans of the index directory. // Keep it up-to-date as we add shards to avoid doing // a filesystem rescan on new shard creation. // - // txtype -> index -> *shardSet - index2shards map[txtype]map[string]*shardSet - - isBlueGreen bool + // index -> *shardSet + index2shards map[string]*shardSet StorageConfig *storage.Config RBFConfig *rbfcfg.Config } -func newIndex2Shards() (r map[txtype]map[string]*shardSet) { - r = make(map[txtype]map[string]*shardSet) +func newIndex2Shards() (r map[string]*shardSet) { + r = make(map[string]*shardSet) return } @@ -350,51 +258,6 @@ func newShardSetFromMap(m map[uint64]bool) *shardSet { } } -// HasData returns true if the database has at least one key. -// For roaring it returns true if we a fragment stored. -// The `which` argument is the index into the per.W slice. 0 for blue, 1 for green. -// If you pass 1, be sure you have a blue-green configuration. -func (per *DBPerShard) HasData(which int) (hasData bool, err error) { - // has to aggregate across all available DBShard for each index and shard. - - if per.types[which] == roaringTxn { - return per.RoaringHasData() // this needs to be made accurate - } - - for _, v := range per.Flatmap { - hasData, err = v.W[which].HasData() - if err != nil { - return - } - if hasData { - return - } - } - return -} - -func (per *DBPerShard) RoaringHasData() (bool, error) { - idxs := per.holder.Indexes() - const requireData = true - for _, idx := range idxs { - shards, err := per.TypedDBPerShardGetShardsForIndex(roaringTxn, idx, "", requireData) - if err != nil { - return false, err - } - if len(shards) > 0 { - return true, nil - } - } - return false, nil -} - -func (per *DBPerShard) ListOpenString() (r string) { - for _, v := range per.Flatmap { - r += v.HolderPath + " -> " + v.W[per.useOpenList].OpenListString() + "\n" - } - return -} - func (per *DBPerShard) LoadExistingDBs() (err error) { idxs := per.holder.Indexes() @@ -414,37 +277,24 @@ func (per *DBPerShard) LoadExistingDBs() (err error) { return } -func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string, holder *Holder) (d *DBPerShard) { +func (txf *TxFactory) NewDBPerShard(typ txtype, holderDir string, holder *Holder) (d *DBPerShard) { if holder.cfg == nil || holder.cfg.RBFConfig == nil || holder.cfg.StorageConfig == nil { PanicOn("must have holder.cfg.RBFConfig and holder.cfg.StorageConfig set here") } - useOpenList := 0 hasRoaring := false - if types[0] == roaringTxn { + if typ == roaringTxn { hasRoaring = true } - if len(types) == 2 { - // blue-green, avoid the empty roaring Tx open list. - // Prefer B's open list if neither is roaring. - if types[0] == roaringTxn || types[1] != roaringTxn { - useOpenList = 1 - } - if types[1] == roaringTxn { - hasRoaring = true - } - } d = &DBPerShard{ - types: types, + typ: typ, HolderDir: holderDir, holder: holder, dbh: NewDBHolder(), Flatmap: make(map[flatkey]*DBShard), txf: txf, - useOpenList: useOpenList, hasRoaring: hasRoaring, - isBlueGreen: len(types) > 1, index2shards: newIndex2Shards(), StorageConfig: holder.cfg.StorageConfig, RBFConfig: holder.cfg.RBFConfig, @@ -469,14 +319,12 @@ func (per *DBPerShard) DeleteIndex(index string) (err error) { if err != nil { return errors.Wrap(err, "DBPerShard.DeleteIndex dbs.Close()") } - for _, ty := range per.types { - path := dbs.pathForType(ty) - err = os.RemoveAll(path) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("DBPerShard.DeleteIndex os.RemoveAll('%v')", path)) - } - delete(per.index2shards[ty], index) + path := dbs.pathForType(per.typ) + err = os.RemoveAll(path) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("DBPerShard.DeleteIndex os.RemoveAll('%v')", path)) } + delete(per.index2shards, index) } // allow the index to be created again anew. @@ -502,10 +350,8 @@ func (per *DBPerShard) DeleteFieldFromStore(index, field, fieldPath string) (err return nil } for _, dbs := range dbi.Shard { - for _, w := range dbs.W { - if e := w.DeleteField(index, field, fieldPath); e != nil && err == nil { - err = errors.Wrap(e, "DeleteFieldFromStore()") - } + if e := dbs.W.DeleteField(index, field, fieldPath); e != nil && err == nil { + err = errors.Wrap(e, "DeleteFieldFromStore()") } } return err @@ -521,46 +367,6 @@ func (per *DBPerShard) DeleteFragment(index, field, view string, shard uint64, f return dbs.DeleteFragment(index, field, view, shard, frag) } -func (dbs *DBShard) DumpAll() { - short := false - fmt.Printf("\n============= begin DumpAll dbs=%p index='%v', shard=%v ========\n", dbs, dbs.Index, int(dbs.Shard)) - for i, ty := range dbs.types { - _ = i - tx, err := dbs.W[i].NewTx(!writable, "", Txo{Index: dbs.idx}) - PanicOn(err) - defer tx.Rollback() - fmt.Printf("\n============= dumping dbs.W[%v] %v ========\n", i, ty) - tx.Dump(short, dbs.Shard) - - switch ty { - case roaringTxn: - case rbfTxn: - case boltTxn: - default: - PanicOn(fmt.Sprintf("unknown txtyp: '%v'", ty)) - } - } - fmt.Printf("\n============= end of DumpAll index='%v', shard=%v ========\n", dbs.Index, int(dbs.Shard)) -} - -func (per *DBPerShard) DumpAll() { - per.Mu.Lock() - defer per.Mu.Unlock() - - found1 := false - for _, dbi := range per.dbh.Index { - for _, dbs := range dbi.Shard { - if dbs.Open { - found1 = true - dbs.DumpAll() - } - } - } - if !found1 { - AlwaysPrintf("DBPerShard.DumpAll() sees no databases. dir='%v'", per.HolderDir) - } -} - // if you know the shard, you can use this // pathForType and prefixForType must be kept in sync! func (dbs *DBShard) pathForType(ty txtype) string { @@ -570,11 +376,6 @@ func (dbs *DBShard) pathForType(ty txtype) string { // is a no-op anyhow. so doesn't need to be correct atm. path := dbs.HolderPath + sep + dbs.Index + sep + backendsDir + sep + ty.DirectoryName() + sep + fmt.Sprintf("shard.%04v", dbs.Shard) - if ty == boltTxn { - // special case: - // bolt doesn't use a directory like the others, just a direct path. - path += sep + "bolt.db" - } return path } @@ -593,23 +394,13 @@ var ErrNoData = fmt.Errorf("no data") // // Caller must hold per.Mu.Lock() already. func (per *DBPerShard) updateIndex2ShardCacheWithNewShard(dbs *DBShard) { - - for _, ty := range dbs.types { - mapIndex2shardSet, ok := per.index2shards[ty] - if !ok { - mapIndex2shardSet = make(map[string]*shardSet) - per.index2shards[ty] = mapIndex2shardSet - } - // INVAR: mapIndex2shardSet is good, but may be an empty map - - shardset, ok := mapIndex2shardSet[dbs.Index] - if !ok { - shardset = newShardSet() - mapIndex2shardSet[dbs.Index] = shardset - } - // INVAR: shardset is present, not nil; a map that can be added to. - shardset.add(dbs.Shard) + shardset, ok := per.index2shards[dbs.Index] + if !ok { + shardset = newShardSet() + per.index2shards[dbs.Index] = shardset } + // INVAR: shardset is present, not nil; a map that can be added to. + shardset.add(dbs.Shard) } func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *DBShard, err error) { @@ -629,76 +420,47 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In } dbs, ok = dbi.Shard[shard] if dbs != nil && dbs.closed { - if len(per.types) == 1 && per.types[0] == roaringTxn { - // roaring txn are nil/fake anyway. Don't freak out. - } else { - PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.types[0]='%v'; len(per.types)=%v", dbs, per.types[0], len(per.types))) + // roaring txn are nil/fake anyway. Don't freak out. + if per.typ != roaringTxn { + PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.typ='%v'", dbs, per.typ)) } } if !ok { dbs = &DBShard{ - types: per.types, + typ: per.typ, ParentDBIndex: dbi, Index: index, Shard: shard, HolderPath: per.HolderDir, idx: idx, per: per, - useOpenList: per.useOpenList, hasRoaring: per.hasRoaring, - isBlueGreen: len(per.types) > 1, } - dbs.stypes = make([]string, len(per.types)) - for i, ty := range per.types { - dbs.stypes[i] = ty.String() - } - + dbs.styp = per.typ.String() dbi.Shard[shard] = dbs per.updateIndex2ShardCacheWithNewShard(dbs) } if !dbs.Open { var registry DBRegistry - for _, ty := range dbs.types { - switch ty { - case roaringTxn: - registry = globalRoaringReg - case rbfTxn: - registry = globalRbfDBReg - registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig) - case boltTxn: - registry = globalBoltReg - default: - PanicOn(fmt.Sprintf("unknown txtyp: '%v'", ty)) - } - path := dbs.pathForType(ty) - w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.StorageConfig) - PanicOn(err) - h := idx.Holder() - w.SetHolder(h) - dbs.Open = true - if w != nil && len(dbs.W) == 0 { - per.Flatmap[flatkey{index: index, shard: shard}] = dbs - } - dbs.W = append(dbs.W, w) + switch dbs.typ { + case roaringTxn: + registry = globalRoaringReg + case rbfTxn: + registry = globalRbfDBReg + registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig) + default: + PanicOn(fmt.Sprintf("unknown txtyp: '%v'", dbs.typ)) } + path := dbs.pathForType(dbs.typ) + w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.StorageConfig) + PanicOn(err) + h := idx.Holder() + w.SetHolder(h) + dbs.Open = true + per.Flatmap[flatkey{index: index, shard: shard}] = dbs + dbs.W = w } - return -} - -func (per *DBPerShard) Del(dbs *DBShard) (err error) { - per.Mu.Lock() - defer per.Mu.Unlock() - - err = dbs.Close() - if err != nil { - return - } - PanicOn(dbs.DeleteDBPath()) - delete(per.Flatmap, flatkey{index: dbs.Index, shard: dbs.Shard}) - - // delete from the heirarchy - delete(dbs.ParentDBIndex.Shard, dbs.Shard) - return nil + return dbs, nil } func (per *DBPerShard) Close() (err error) { @@ -718,28 +480,7 @@ func (per *DBPerShard) Close() (err error) { // If requireData, we open the database and see that it has a key, rather // than assume that the database file presence is enough. func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requireData bool) (map[uint64]bool, error) { - - n := len(f.types) - if n != 1 && n != 2 { - PanicOn(fmt.Sprintf("internal error. only green or blue/green supported. we see types len %v", n)) - } - - var shards []map[uint64]bool - for _, ty := range f.types { - ss, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(ty, idx, roaringViewPath, requireData) - if err != nil { - return nil, err - } - shards = append(shards, ss) - } - - // Note: we don't actually know when the blue call and when the green call comes - // through here. So if we are deleting a shard, we will see a difference earlier - // in one than the other. TestAPI_ClearFlagForImportAndImportValues for example. - // Hence we cannot do a blue-green check here for matching shards. - - // If we are populating blue from green, it does matter that we return green. - return shards[n-1], nil + return f.dbPerShard.TypedDBPerShardGetShardsForIndex(f.typ, idx, roaringViewPath, requireData) } // if roaringViewPath is "" then for ty == roaringTxn we go to disk to discover @@ -751,10 +492,6 @@ func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requir // when a new DBShard is made, we will update the list of shards then. Thus // the per.index2shard should always be up to date AFTER the first call here. // -// Note: we cannot here call GetView2ShardsMapForIndex() because that only ever -// returns the green data and we are used during migration for both blue -// and green. -// func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, roaringViewPath string, requireData bool) (shardMap map[uint64]bool, err error) { // use the cache, always @@ -769,13 +506,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r return shardMap, nil } - i2ss, ok := per.index2shards[ty] - if !ok { - // index -> shardSet - i2ss = make(map[string]*shardSet) - per.index2shards[ty] = i2ss - } - // INVAR: i2ss is good, but may be an empty map + i2ss := per.index2shards ss, ok := i2ss[idx.name] if ok { @@ -785,7 +516,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r // gotta read shards from disk directory layout. setOfShards := newShardSet() - per.index2shards[ty][idx.name] = setOfShards + per.index2shards[idx.name] = setOfShards // Upon return, cache the setOfShards value and reuse it next time @@ -854,13 +585,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r } func (per *DBPerShard) unprotectedTypedIndexShardHasData(ty txtype, idx *Index, shard uint64) (hasData bool, err error) { - whichty := 0 - if len(per.types) == 2 { - if ty == per.types[1] { - whichty = 1 - } - } - if ty != per.types[whichty] { + if ty != per.typ { return } @@ -871,7 +596,7 @@ func (per *DBPerShard) unprotectedTypedIndexShardHasData(ty txtype, idx *Index, "per.GetDBShard(index='%v', shard='%v', ty='%v')", idx.name, shard, ty.String())) } - return dbs.W[whichty].HasData() + return dbs.W.HasData() } func listDirUnderDir(root string, includeRoot bool, ignoreEmpty bool) (files []string, err error) { @@ -906,224 +631,6 @@ func listDirUnderDir(root string, includeRoot bool, ignoreEmpty bool) (files []s return } -// populateBlueFromGreen prepares for a blue_green run at startup time. -// -// It is called at the end of Holder.Open(). This allows the application -// of blue-green checking to pilosa instances that -// were previously run only with a single (solo) backend. -// -// PRE: This operation requires, at its start, either: -// -// (1) an empty blue database -- this allows transitioning from -// a solo database to blue_green checking where the solo -// becomes the green; or -// -// (2) that the blue data, if present, be logically -// identical to the green data -- this allows one to restart -// a pilosa that was already running in blue_green mode -// and remain in blue_green mode. -// -// In either case, the goal to to finish populateBlueFromGreen() -// and have the exact same logical set of data in both backends. -// -// Why must the data be identical after Holder.Open() finishes? -// Otherwise subsequent blue-green checks have no hope of -// being accurate. -// -// The blue is the destination -- this is always types[0]. -// The green source is always types[1]. The mnemonic is blue_geen. -// The blue is first, so it is in types[0]. The green -// is second, in types[1]. For example, with PILOSA_STORAGE_BACKEND=bolt_roaring -// we have bolt as blue, and roaring as green. The contents of -// bolt must be empty or exactly match roaring. If bolt -// starts empty, it will be populated from roaring by -// populateBlueFromGreen(). -// -func (dbs *DBShard) populateBlueFromGreen() (err error) { - - n := len(dbs.W) - if n != 2 { - PanicOn(fmt.Sprintf("populateBlueFromGreen did not find 2 open DBs: have %v", n)) - } - - dest := dbs.W[0] // blue - src := dbs.W[1] // green - - // copy all the key/container pairs. - // Since a shard is fairly small, we think one Tx will suffice. - - readtx, err := src.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard}) - PanicOn(err) - defer readtx.Rollback() - - writetx, err := dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard}) - PanicOn(err) - defer writetx.Rollback() - - ctWriteCount := 0 - - for _, fld := range dbs.idx.Fields() { - field := fld.Name() - for _, vw := range fld.views() { - view := vw.name - citer, _, err := readtx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0) - if err != nil { - // might be an empty fragment. If so, let's not freak out. - if strings.Contains(err.Error(), "fragment not found") { - continue - } else { - writetx.Rollback() - return errors.Wrap(err, "DBShard.populateBlueFromGreen readtx.ContainerIterator") - } - } - - for citer.Next() { - ckey, rc := citer.Value() - err := writetx.PutContainer(dbs.Index, field, view, dbs.Shard, ckey, rc) - if err != nil { - citer.Close() - writetx.Rollback() - return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.PutContainer") - } - - ctWriteCount++ - if ctWriteCount%1000 == 1 { - - // regularly commiting smaller batches and the first batch as soon as - // possible massively speeds up writing to bolt. - // - // reference: https://github.com/boltdb/bolt/issues/94 - // - // benbjohnson commented on Mar 25, 2014 - // "Bulk loading more than 1000 items at a time is very slow. This is because nodes - // are not splitting before commit which causes large memmove() operations during insertion." - // runtime.memmove is taking all of the time in our pprof profile, when copying rbf to bolt, so we suspect it is this. - // - err = writetx.Commit() - if err != nil { - citer.Close() - writetx.Rollback() - return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.Commit") - } - writetx, err = dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard}) - if err != nil { - citer.Close() - writetx.Rollback() - return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.NewTx inside citer.Next() loop") - } - } - - } - citer.Close() - } - } - err = writetx.Commit() - if err != nil { - return errors.Wrap(err, "writetx.Commit()") - } - return nil -} - -// verifyBlueEqualsGreen checks that blue and green are identical. -func (dbs *DBShard) verifyBlueEqualsGreen() (err error) { - - n := len(dbs.W) - if n != 2 { - PanicOn(fmt.Sprintf("verifyBlueEqualsGreen did not find 2 open DBs: have %v", n)) - } - - blue := dbs.W[0] - green := dbs.W[1] - - greentx, err := green.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard}) - PanicOn(err) - defer greentx.Rollback() - - bluetx, err := blue.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard}) - PanicOn(err) - defer bluetx.Rollback() - - for _, fld := range dbs.idx.Fields() { - field := fld.Name() - for _, vw := range fld.views() { - - view := vw.name - gCiter, _, err := greentx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0) - if err != nil { - if strings.Contains(err.Error(), "fragment not found") { - continue - } else { - return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen greentx.ContainerIterator") - } - } - - bCiter, _, err := bluetx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0) - if err != nil { - gCiter.Close() - if bCiter != nil { - bCiter.Close() - } - return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen bluetx.ContainerIterator") - } - - for gCiter.Next() { - greenCkey, greenc := gCiter.Value() - - if !bCiter.Next() { - bCiter.Close() - gCiter.Close() - return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ - "sees missing blue container at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v' the greenCkey: '%v'", - dbs.Index, field, view, dbs.Shard, greenCkey)) - } - blueCkey, bluec := bCiter.Value() - - if blueCkey != greenCkey { - bCiter.Close() - gCiter.Close() - return fmt.Errorf("DBShard.verifyBlueEqualsGreen sees sequence-of-ckey "+ - "difference: blueCkey %v not equal to greenCkey %v at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v'", - blueCkey, greenCkey, dbs.Index, field, view, dbs.Shard) - } - nGreen := greenc.N() - nBlue := bluec.N() - if nBlue != nGreen { - bCiter.Close() - gCiter.Close() - return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ - "sees variation in blue at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v", - dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue)) - } - err = bluec.BitwiseCompare(greenc) - if err != nil { - bCiter.Close() - gCiter.Close() - return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ - "sees variation in blue at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v ; BitwiseCompare response: '%v'", - dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue, err)) - } - } - if bCiter.Next() { - blueCkey, _ := bCiter.Value() - bCiter.Close() - gCiter.Close() - return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+ - "sees extra blue container (not present in green) at index: '%v' field: '%v' view: '%v' "+ - "shard: '%v' the ckey: '%v'", - dbs.Index, field, view, dbs.Shard, blueCkey)) - } - bCiter.Close() - gCiter.Close() - } - } - - return nil -} - type FieldView2Shards struct { // field -> view -> *shardSet m map[string]map[string]*shardSet @@ -1232,15 +739,8 @@ func (vs *FieldView2Shards) removeField(name string) { delete(vs.m, name) } -// Note: cannot call this during migration, because -// it only ever returns the green shards if we are in blue-green. func (per *DBPerShard) GetFieldView2ShardsMapForIndex(idx *Index) (vs *FieldView2Shards, err error) { - - // for blue-green, it does matter that we return green, so we can migrate from it. - ty := per.types[0] - if per.isBlueGreen { - ty = per.types[1] - } + ty := per.typ switch ty { case roaringTxn: diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go index 9482d1504..13c561810 100644 --- a/dbshard_internal_test.go +++ b/dbshard_internal_test.go @@ -84,7 +84,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { v2s.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet) } - for _, src := range []string{"roaring", "bolt", "rbf"} { + for _, src := range []string{"roaring", "rbf"} { cfg := mustHolderConfig() cfg.StorageConfig.Backend = src holder := NewHolder(tmpdir, cfg) @@ -145,7 +145,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) { tx.Rollback() } } else { - // non-roaring: rbf, bolt + // non-roaring: rbf for _, shard := range []uint64{93, 223, 221, 215, 219, 217} { tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard}) @@ -191,14 +191,6 @@ rick/fields/_exists/views/standard/fragments/217 rick/fields/_exists/views/standard/fragments/93 rick/fields/_exists/views/standard/fragments/219 rick/fields/_exists/views/standard/fragments/223 -`, - "bolt": ` -rick/backends/backend-boltdb/shard.0093-bolt/bolt.db -rick/backends/backend-boltdb/shard.0215-bolt/bolt.db -rick/backends/backend-boltdb/shard.0217-bolt/bolt.db -rick/backends/backend-boltdb/shard.0219-bolt/bolt.db -rick/backends/backend-boltdb/shard.0221-bolt/bolt.db -rick/backends/backend-boltdb/shard.0223-bolt/bolt.db `, "rbf": ` rick/backends/backend-rbf/shard.0093-rbf @@ -225,7 +217,7 @@ func makeSampleRoaringDir(t *testing.T, root, index, backend string, minBytes in } var shard uint64 switch backend { - case "bolt", "rbf": + case "rbf": shard = shards[i] idx = helperCreateDBShard(h, index, shard) @@ -266,16 +258,8 @@ func helperCreateDBShard(h *Holder, index string, shard uint64) *Index { } // keep the ocd linter happy -var _ = makeBolttestDB var _ = makeRBFtestDB -func makeBolttestDB(path string, h *Holder, shard uint64) { - i := uint64(1) - w, _ := mustOpenEmptyBoltWrapper(path) - BoltMustSetBitvalue(w, "index", "field", "view", shard, i) - w.Close() -} - func makeRBFtestDB(path string, h *Holder, shard uint64) { i := uint64(1) diff --git a/delete_test.go b/delete_test.go index 061b72d1d..886b71e81 100644 --- a/delete_test.go +++ b/delete_test.go @@ -27,7 +27,6 @@ import ( ) func TestExecutor_DeleteRecords(t *testing.T) { - pilosa.NotBlueGreenTest(t) indexName := "i" setup := func(t *testing.T, r *require.Assertions, c *test.Cluster) { t.Helper() diff --git a/executor_test.go b/executor_test.go index adcf64dbb..ba357d724 100644 --- a/executor_test.go +++ b/executor_test.go @@ -3843,7 +3843,6 @@ func TestExecutor_Execute_Existence(t *testing.T) { hldr2 := c.GetHolder(0) index2 := hldr2.Index("i") _ = index2 - //index2.Dump("after reopen") if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil { t.Fatal(err) diff --git a/fragment.go b/fragment.go index daab8bb34..d3c52c7dc 100644 --- a/fragment.go +++ b/fragment.go @@ -47,6 +47,7 @@ import ( "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/shardwidth" "github.com/molecula/featurebase/v2/stats" + "github.com/molecula/featurebase/v2/storage" "github.com/molecula/featurebase/v2/testhook" "github.com/molecula/featurebase/v2/topology" "github.com/molecula/featurebase/v2/tracing" @@ -75,9 +76,6 @@ const ( // snapshotExt is the file extension used for an in-process snapshot. snapshotExt = ".snapshotting" - // copyExt is the file extension used for the temp file used while copying. - copyExt = ".copying" - // cacheExt is the file extension for persisted cache ids. cacheExt = ".cache" @@ -441,7 +439,7 @@ func (f *fragment) inspectStorage(data []byte, file *os.File, newGen generation, // (remapping an existing bitmap to match a new backing store). func (f *fragment) openStorage(unmarshalData bool) error { - useRowCache := f.idx.Txf().UseRowCache() + useRowCache := storage.RowCacheEnabled() if !f.idx.NeedsSnapshot() { f.gen = &NopGeneration{} if useRowCache { @@ -626,7 +624,7 @@ func (f *fragment) mustRow(tx Tx, rowID uint64) *Row { // (updating the cache). func (f *fragment) unprotectedRow(tx Tx, rowID uint64) (*Row, error) { - useRowCache := tx.UseRowCache() + useRowCache := storage.RowCacheEnabled() if useRowCache { if f.rowCache == nil { f.rowCache = newSimpleCache() @@ -699,7 +697,7 @@ func (f *fragment) setBit(tx Tx, rowID, columnID uint64) (changed bool, err erro if tx.Type() == RoaringTxn { return changed, errors.New("internal error: f.gen was nil and tx.Type is RoaringTxn - should never happen under roaring b/c storage should be open") } - // else blue green or transactional backend. Just do it. + // else transactional backend. Just do it. err = doSetFunc() } return changed, err @@ -744,7 +742,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo delete(f.checksums, int(rowID/HashBlockSize)) // Increment number of operations until snapshot is required. - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, 1) + f.incrementOpN(1) // If we're using a cache, update it. Otherwise skip the // possibly-expensive count operation. @@ -757,7 +755,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo } // Drop the rowCache entry; it's wrong, and we don't want to force // a new copy if no one's reading it. - if tx.UseRowCache() && f.rowCache != nil { + if storage.RowCacheEnabled() && f.rowCache != nil { f.rowCache.Add(rowID, nil) } @@ -809,7 +807,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b delete(f.checksums, int(rowID/HashBlockSize)) // Increment number of operations until snapshot is required. - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, 1) + f.incrementOpN(1) // If we're using a cache, update it. Otherwise skip the // possibly-expensive count operation. @@ -822,7 +820,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b } // Drop the rowCache entry; it's wrong, and we don't want to force // a new copy if no one's reading it. - if tx.UseRowCache() && f.rowCache != nil { + if storage.RowCacheEnabled() && f.rowCache != nil { f.rowCache.Add(rowID, nil) } @@ -891,7 +889,7 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo } // invalidate rowCache for this row. - if tx.UseRowCache() && f.rowCache != nil { + if storage.RowCacheEnabled() && f.rowCache != nil { f.rowCache.Add(rowID, nil) } @@ -942,7 +940,7 @@ func (f *fragment) unprotectedClearRow(tx Tx, rowID uint64) (changed bool, err e // Clear the row in cache. f.cache.Add(rowID, 0) - if tx.UseRowCache() && f.rowCache != nil { + if storage.RowCacheEnabled() && f.rowCache != nil { f.rowCache.Add(rowID, nil) } @@ -2426,7 +2424,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64 if f.storage != nil { wp = &f.storage.OpWriter } - useRowCache := tx.UseRowCache() + useRowCache := storage.RowCacheEnabled() doFunc := func() error { if len(set) > 0 { @@ -2438,7 +2436,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64 return errors.Wrap(err, "adding positions") } f.stats.Count(MetricImportedN, int64(changedN), 1) - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, changedN) + f.incrementOpN(changedN) } if len(clear) > 0 { @@ -2448,7 +2446,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64 return errors.Wrap(err, "clearing positions") } f.stats.Count(MetricClearedN, int64(changedN), 1) - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, changedN) + f.incrementOpN(changedN) } // Update cache counts for all affected rows. @@ -2735,7 +2733,7 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b rowSize := uint64(1 << shardVsContainerExponent) span, ctx := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits") - useRowCache := tx.UseRowCache() + useRowCache := storage.RowCacheEnabled() var changed int var rowSet map[uint64]int var wp *io.Writer @@ -2749,7 +2747,7 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b return err } - changed, rowSet, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, rit, clear, true, rowSize, nil) + changed, rowSet, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, rit, clear, true, rowSize) return err }) @@ -2789,7 +2787,7 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b span, _ = tracing.StartSpanFromContext(ctx, "importRoaring.incrementOpN") - tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, changed) + f.incrementOpN(changed) span.Finish() return nil @@ -2815,6 +2813,10 @@ func (f *fragment) incrementOpN(changed int) { if changed <= 0 { return } + // don't count opN or ops if our index doesn't want snapshots + if !f.idx.NeedsSnapshot() { + return + } f.opN += changed f.ops++ if f.opN > f.MaxOpN { @@ -3015,11 +3017,15 @@ func (f *fragment) writeStorageToArchive(tw *tar.Writer) error { tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Shard: f.shard}) defer tx.Rollback() - file, sz, err := tx.RoaringBitmapReader(f.index(), f.field(), f.view(), f.shard, f.path()) + rbm, err := tx.RoaringBitmap(f.index(), f.field(), f.view(), f.shard) if err != nil { - return err + return errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") + } + var buf bytes.Buffer + sz, err := rbm.WriteTo(&buf) + if err != nil { + return errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") } - defer file.Close() // Write archive header. if err := tw.WriteHeader(&tar.Header{ @@ -3033,7 +3039,7 @@ func (f *fragment) writeStorageToArchive(tw *tar.Writer) error { // Copy the file up to the last known size. // This is done outside the lock because the storage format is append-only. - if _, err := io.CopyN(tw, file, sz); err != nil { + if _, err := io.CopyN(tw, &buf, sz); err != nil { return errors.Wrap(err, "copying") } return nil @@ -3086,8 +3092,7 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) { // Process file based on file name. switch hdr.Name { case "data": - idx := f.holder.Index(f.index()) - tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard}) + tx := f.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Fragment: f, Shard: f.shard}) defer tx.Rollback() if err := f.fillFragmentFromArchive(tx, tr); err != nil { return 0, errors.Wrap(err, "reading storage") @@ -3131,7 +3136,7 @@ func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error { if err != nil { return errors.Wrap(err, "fillFragmentFromArchive NewRoaringIterator") } - changed, rowSet, err := tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, clear, log, rowSize, data) + changed, rowSet, err := tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, clear, log, rowSize) _, _ = changed, rowSet if err != nil { return errors.Wrap(err, "fillFragmentFromArchive ImportRoaringBits") @@ -3139,40 +3144,6 @@ func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error { return nil } -func (f *fragment) readStorageFromArchive(r io.Reader) error { - - // Create a temporary file to copy into. - path := f.path() + copyExt - file, err := os.Create(path) - if err != nil { - return errors.Wrap(err, "creating directory") - } - defer file.Close() - - // Copy reader into temporary path. - if _, err = io.Copy(file, r); err != nil { - return errors.Wrap(err, "copying") - } - - // TODO(jea): isn't this next Rename a file handle leak? - // try closing first - if err := f.closeStorage(); err != nil { - return errors.Wrap(err, "closeStorage-prior-to-Rename-and-openStorage") - } - - // Move snapshot to data file location. - if err := os.Rename(path, f.path()); err != nil { - return errors.Wrap(err, "renaming") - } - - // Reopen storage. - if err := f.openStorage(true); err != nil { - return errors.Wrap(err, "opening") - } - - return nil -} - func (f *fragment) readCacheFromArchive(r io.Reader) error { // Slurp data from reader and write to disk. buf, err := ioutil.ReadAll(r) @@ -3369,7 +3340,7 @@ func (f *fragment) intRowIterator(tx Tx, wrap bool, filters ...roaring.BitmapFil // accumulator [column ID] -> [int value] acc := make(map[uint64]int64) - if tx.UseRowCache() { + if storage.RowCacheEnabled() { // needs a write lock since it will update the f.rowCache f.mu.Lock() defer f.mu.Unlock() diff --git a/fragment_internal_test.go b/fragment_internal_test.go index 9b4ac461a..23c2063f1 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -183,7 +183,6 @@ func TestFragment_RowcacheMap(t *testing.T) { // Ensure a fragment can clear a row. func TestFragment_ClearRow(t *testing.T) { - NotBlueGreenTest(t) f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") _ = idx defer f.Clean(t) @@ -215,7 +214,6 @@ func TestFragment_ClearRow(t *testing.T) { // Ensure a fragment can set a row. func TestFragment_SetRow(t *testing.T) { - NotBlueGreenTest(t) f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 7, "") _ = idx defer f.Clean(t) @@ -1728,7 +1726,7 @@ func roaringOnlyBenchmark(b *testing.B) { // Ensure a fragment can be copied to another fragment. func TestFragment_WriteTo_ReadFrom(t *testing.T) { - roaringOnlyTest(t) + // roaringOnlyTest(t) f0, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") defer f0.Clean(t) @@ -1741,6 +1739,10 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } else if _, err := f0.clearBit(tx, 1000, 1); err != nil { t.Fatal(err) } + err := tx.Commit() + if err != nil { + t.Fatalf("committing write: %v", err) + } // Verify cache is populated. if n := f0.cache.Len(); n != 1 { @@ -1755,7 +1757,9 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Read into another fragment. - f1, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") + f1, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "") + tx.Rollback() + defer f1.Clean(t) if rn, err := f1.ReadFrom(&buf); err != nil { // eventually calls fragment.fillFragmentFromArchive @@ -1763,6 +1767,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } else if wn != rn { t.Fatalf("read/write byte count mismatch: wn=%d, rn=%d", wn, rn) } + // make a read-only Tx after ReadFrom has committed. + tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f1, Shard: f1.shard}) // Verify cache is in other fragment. if n := f1.cache.Len(); n != 1 { @@ -3049,7 +3055,7 @@ func BenchmarkImportRoaringUpdate(b *testing.B) { // to generate an op log and/or snapshot. itr, err := roaring.NewRoaringIterator(data) PanicOn(err) - _, _, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, false, false, 0, nil) + _, _, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, false, false, 0) if err != nil { b.Errorf("import error: %v", err) } @@ -5215,15 +5221,13 @@ func TestImportValueConcurrent(t *testing.T) { // we will be making a new Tx each time, so we can rollback the default provided one. tx.Rollback() - types := idx.holder.txf.TxTypes() - for _, ty := range types { - switch ty { - case roaringTxn: - t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " + - "blueGreenTx because the lack of transactional consistency " + - "from Roaring-per-file will create false comparison " + - "failures.")) - } + ty := idx.holder.txf.TxTyp() + switch ty { + case roaringTxn: + t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " + + "roaring because the lack of transactional consistency " + + "from Roaring-per-file will create false comparison " + + "failures.")) } eg := &errgroup.Group{} @@ -5364,11 +5368,6 @@ func TestImportValueRowCache(t *testing.T) { // do we see races/corruption around concurrent read/write. // especially on writes to the row cache. func TestFragmentConcurrentReadWrite(t *testing.T) { - // actual transaction backends, there won't be any - // data, and in particular, the blue-green tests will - // note this and fire a false-positive. - NotBlueGreenTest(t) - f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked) defer f.Clean(t) tx.Rollback() @@ -5528,12 +5527,6 @@ func TestFragment_Bug_Q2DoubleDelete(t *testing.T) { } } -func NotBlueGreenTest(t *testing.T) { - if strings.Contains(CurrentBackend(), "_") { - t.Skip("skip under blue green") - } -} - var mutexSamplesPrepared sync.Once func requireMutexSampleData(tb testing.TB) { diff --git a/go.mod b/go.mod index ff7c1d715..67d9c30bd 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,6 @@ require ( github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect - github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 github.com/go-test/deep v1.0.7 github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.3.3 diff --git a/go.sum b/go.sum index 0959b010c..60db445b5 100644 --- a/go.sum +++ b/go.sum @@ -86,10 +86,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E= -github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= diff --git a/holder.go b/holder.go index dea18e3e7..756d75443 100644 --- a/holder.go +++ b/holder.go @@ -302,7 +302,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { txf, err := NewTxFactory(cfg.StorageConfig.Backend, h.IndexesPath(), h) PanicOn(err) h.txf = txf - h.txf.blueGreenOffIfRunningBlueGreen() _ = testhook.Created(h.Auditor, h, nil) return h @@ -605,8 +604,6 @@ func (h *Holder) Open() error { h.txf = txf } - h.txf.blueGreenOffIfRunningBlueGreen() - // Reset closing in case Holder is being reopened. h.closing = make(chan struct{}) @@ -713,13 +710,6 @@ func (h *Holder) Open() error { return errors.Wrap(err, "Holder.Open h.txf.Open()") } - // under blue_green, we must sync blue from green before we turn on checking. - if err := h.txf.green2blue(h); err != nil { - return errors.Wrap(err, "Holder.Open h.txf.green2blue(h)") - } - - h.txf.blueGreenOnIfRunningBlueGreen() - if h.cfg.LookupDBDSN != "" { h.Logger.Printf("connecting to lookup database") @@ -814,9 +804,6 @@ func (h *Holder) Close() error { if globalUseStatTx { fmt.Printf("%v\n", globalCallStats.report()) } - if h.txf != nil && h.txf.blueGreenReg != nil { - h.txf.blueGreenReg.Close() - } h.Stats.Close() @@ -2131,12 +2118,6 @@ func (h *Holder) addIndex(idx *Index) { h.imu.Unlock() } -func (h *Holder) DumpAllShards() { - h.mu.RLock() - defer h.mu.RUnlock() - h.txf.dbPerShard.DumpAll() -} - func (h *Holder) Txf() *TxFactory { h.mu.Lock() defer h.mu.Unlock() diff --git a/holder_internal_test.go b/holder_internal_test.go index bd35f0b03..7cd8ad630 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -22,7 +22,6 @@ import ( "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/testhook" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck ) var _ = fmt.Printf @@ -109,73 +108,6 @@ func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID ui } } -func testMustHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) { - - //shard := columnID / ShardWidth - - // hmm... if its a new holder, meta data isn't there, so ask for it. - idx, err := h.CreateIndexIfNotExists(index, IndexOptions{}) - PanicOn(err) - - f := idx.Field(field) - if f == nil { - t.Fatalf("no such field '%v'", field) - } - - row, err := f.Row(nil, rowID) - if err != nil { - t.Fatalf("error getting field.Row(rowID=%v): %v", rowID, err) - } - - cols := row.Columns() - if len(cols) == 0 { - t.Fatalf("error getting field.Row().Columns(): empty columns, colID %v bit was not hot", columnID) - } - - for _, c := range cols { - if c == columnID { - return // ok, found it. - } - } - t.Fatalf("error getting field.Row().Columns(): colID %v bit was not hot", columnID) -} - -func testMustNotHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) { - if testHasBit(t, h, index, field, rowID, columnID) { - t.Fatalf("error, expected no bit but this bit was hot: index='%v', field='%v', rowID='%v', columnID='%v'", index, field, rowID, columnID) - } -} - -func testHasBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) bool { - - idx := h.Index(index) - if idx == nil { - return false // not even an index by this name. Obviously no hot bits either. - } - - f := idx.Field(field) - if f == nil { - return false - } - - row, err := f.Row(nil, rowID) - if err != nil { - return false - } - - cols := row.Columns() - if len(cols) == 0 { - return false - } - - for _, c := range cols { - if c == columnID { - return true // ok, found it. - } - } - return false -} - func TestHolderOperatorProcess(t *testing.T) { h, path, err := makeHolder(t, "") if err != nil { diff --git a/index.go b/index.go index 4cfd301b2..8506184f5 100644 --- a/index.go +++ b/index.go @@ -27,7 +27,6 @@ import ( "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/stats" "github.com/molecula/featurebase/v2/testhook" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck "github.com/pkg/errors" "golang.org/x/sync/errgroup" ) @@ -464,7 +463,6 @@ func (i *Index) Close() error { // make it clear what the Index.AvailableShards() calls are trying to obtain. const includeRemote = false -const localOnly = true // AvailableShards returns a bitmap of all shards with data in the index. func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap { @@ -853,14 +851,6 @@ func FormatQualifiedIndexName(index string) string { return fmt.Sprintf("%s\x00", index) } -// Dump prints to stdout the contents of the roaring Containers -// stored in idx. Mostly for debugging. -func (i *Index) Dump(label string) { - fileline := FileLine(2) - fmt.Printf("\n%v Dump: %v\n\n", fileline, label) - i.holder.txf.dbPerShard.DumpAll() -} - func (i *Index) Txf() *TxFactory { return i.holder.txf } diff --git a/pjobs.go b/pjobs.go deleted file mode 100644 index 0636fbd36..000000000 --- a/pjobs.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "sync" - - "github.com/glycerine/idem" -) - -// parallelJobs runs functions in parallel on a goroutine -// pool that has nGoro goroutines. -type parallelJobs struct { - nGoro int - - jobQ chan func(worker int) error - halters []*idem.Halter - - // err is protected by errmu - err error - errmu sync.Mutex -} - -func newParallelJobs(nGoro int) (p *parallelJobs) { - if nGoro < 1 { - // 0 really means, - // "turn it up to 11". - // same for negative. - nGoro = 10000 - } - // maximum 10K goroutines - if nGoro > 10000 { - nGoro = 10000 - } - - p = ¶llelJobs{ - nGoro: nGoro, - jobQ: make(chan func(worker int) error, 10000), - halters: make([]*idem.Halter, nGoro), - } - - for j := 0; j < nGoro; j++ { - h := idem.NewHalter() - p.halters[j] = h - } - - for i, h := range p.halters { - go func(h *idem.Halter, worker int) { - defer h.MarkDone() - for { - select { - case <-h.ReqStop.Chan: - return - case f, ok := <-p.jobQ: - if !ok { - // channel closed, finish up - return - } - - err1 := f(worker) - if err1 != nil { - p.errmu.Lock() - if p.err == nil { - p.err = err1 - } - p.errmu.Unlock() - // an error occurred, tell everyone to stop - for _, h2 := range p.halters { - h2.RequestStop() - } - return - } - } - } - }(h, i) - } - return -} - -// return value accepted will be false if we are shutting down -// due to an error. -func (p *parallelJobs) run(fun func(worker int) error) (accepted bool) { - select { - case <-p.halters[0].ReqStop.Chan: - return false - case p.jobQ <- fun: - return true - } -} - -func (p *parallelJobs) waitForFinish() error { - - // tell the workers no more jobs. - close(p.jobQ) - - // wait for everyone to finish - for i, h := range p.halters { - _ = i - <-h.Done.Chan - } - - return p.err -} diff --git a/pjobs_test.go b/pjobs_test.go deleted file mode 100644 index 71192b278..000000000 --- a/pjobs_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pilosa - -import ( - "fmt" - "sync/atomic" - "testing" -) - -func Test_ParallelJobs_EarlyShutdown_WaitsForAllGoro(t *testing.T) { - const n = 10000 // total jobs to run - - var errLastOne = fmt.Errorf("the last job has run, and returned this error") - pj := newParallelJobs(100) - nTotal := int64(0) - for i := 0; i < n; i++ { - accepted := pj.run(func(worker int) error { - highpoint := atomic.AddInt64(&nTotal, 1) - switch int(highpoint) { - case n - 1: - return errLastOne - } - return nil - }) - if !accepted { - panic("should have been accepted") - } - } - err := pj.waitForFinish() - tot := atomic.LoadInt64(&nTotal) - if int(tot) != n { - panic(fmt.Sprintf("We didn't run them all? tot=%v, n=%v; pj.jobQ len %v; err='%v'", tot, n, len(pj.jobQ), err)) - } - if err != errLastOne { - panic("expected to see errLastOne") - } - // good: finished cleanly. -} diff --git a/rbf.go b/rbf.go index 64a268281..cee25f35c 100644 --- a/rbf.go +++ b/rbf.go @@ -15,15 +15,12 @@ package pilosa import ( - "bytes" "fmt" "io" - "io/ioutil" "math" "os" "strings" "sync" - "sync/atomic" "github.com/molecula/featurebase/v2/rbf" rbfcfg "github.com/molecula/featurebase/v2/rbf/cfg" @@ -76,8 +73,6 @@ func (w *RbfDBWrapper) CleanupTx(tx Tx) { w.muDb.Lock() delete(w.openTx, r) - //vv("rbf CleanupTx gid %v about to call r.o.dbs.Cleanup(tx.Sn=%v)", curGID(), tx.Sn()) - r.o.dbs.Cleanup(tx) // release the read/write lock. w.muDb.Unlock() } @@ -186,20 +181,12 @@ type RBFTx struct { initialIndex string tx *rbf.Tx o Txo - sn int64 // serial number Db *RbfDBWrapper done bool mu sync.Mutex // protect done as it changes state } -func (tx *RBFTx) IsDone() (done bool) { - tx.mu.Lock() - done = tx.done - tx.mu.Unlock() - return -} - func (tx *RBFTx) DBPath() string { return tx.tx.DBPath() } @@ -210,17 +197,13 @@ func (tx *RBFTx) Type() string { func (tx *RBFTx) Rollback() { tx.tx.Rollback() - - // must happen after actual rollback tx.Db.CleanupTx(tx) } func (tx *RBFTx) Commit() (err error) { err = tx.tx.Commit() - - // must happen after actual commit tx.Db.CleanupTx(tx) - return + return err } func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) { @@ -412,10 +395,6 @@ func (tx *RBFTx) Min(index, field, view string, shard uint64) (uint64, bool, err return tx.tx.Min(rbfName(index, field, view, shard)) } -func (tx *RBFTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - return tx.tx.UnionInPlace(rbfName(index, field, view, shard), others...) -} - // CountRange returns the count of hot bits in the start, end range on the fragment. // roaring.countRange counts the number of bits set between [start, end). func (tx *RBFTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { @@ -426,24 +405,8 @@ func (tx *RBFTx) OffsetRange(index, field, view string, shard uint64, offset, st return tx.tx.OffsetRange(rbfName(index, field, view, shard), offset, start, end) } -func (tx *RBFTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {} - -func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { - return tx.tx.ImportRoaringBits(rbfName(index, field, view, shard), rit, clear, log, rowSize, data) -} - -func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - - rbm, err := tx.RoaringBitmap(index, field, view, shard) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap") - } - var buf bytes.Buffer - sz, err = rbm.WriteTo(&buf) - if err != nil { - return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)") - } - return ioutil.NopCloser(&buf), sz, err +func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { + return tx.tx.ImportRoaringBits(rbfName(index, field, view, shard), rit, clear, log, rowSize) } func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { @@ -452,39 +415,6 @@ func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring return b.Iterator() } -func (tx *RBFTx) Pointer() string { - return fmt.Sprintf("%p", tx) -} - -func (tx *RBFTx) Dump(short bool, shard uint64) { - tx.tx.Dump(short, shard) -} - -// Readonly is true if the transaction is not read-and-write, but only doing reads. -func (tx *RBFTx) Readonly() bool { - return !tx.tx.Writable() -} - -func (tx *RBFTx) Group() *TxGroup { - return tx.o.Group -} - -func (tx *RBFTx) Options() Txo { - return tx.o -} - -func (tx *RBFTx) Sn() int64 { - return tx.sn -} - -func (tx *RBFTx) UseRowCache() bool { - // since RFB returns memory mapped data, we can't use - // the rowCache without first making a copy. - // So we only use the rowCache if the copy is - // enabled. - return storage.EnableRowCache() -} - func (tx *RBFTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { return tx.tx.ApplyFilter(rbfName(index, field, view, shard), ckey, filter) } @@ -591,20 +521,16 @@ func (w *RbfDBWrapper) OpenDB() error { return nil } -var globalNextTxSnRBFTx int64 - func (w *RbfDBWrapper) NewTx(write bool, initialIndex string, o Txo) (_ Tx, err error) { tx, err := w.db.Begin(write) if err != nil { return nil, err } - sn := atomic.AddInt64(&globalNextTxSnRBFTx, 1) rtx := &RBFTx{ tx: tx, initialIndex: initialIndex, o: o, - sn: sn, Db: w, } @@ -629,20 +555,6 @@ func (w *RbfDBWrapper) DeleteFragment(index, field, view string, shard uint64, f return tx.Commit() } -func (w *RbfDBWrapper) DeleteDBPath(dbs *DBShard) error { - path := dbs.pathForType(rbfTxn) - return os.RemoveAll(path) -} - func (w *RbfDBWrapper) OpenListString() (r string) { return "rbf OpenListString not implemented yet" } - -func (w *RbfDBWrapper) OpenSnList() (slc []int64) { - w.muDb.Lock() - for v := range w.openTx { - slc = append(slc, v.sn) - } - w.muDb.Unlock() - return -} diff --git a/rbf/cursor_internal_test.go b/rbf/cursor_internal_test.go index 3d8dca800..e426dda5b 100644 --- a/rbf/cursor_internal_test.go +++ b/rbf/cursor_internal_test.go @@ -59,20 +59,12 @@ func TestCursor_RoaringImport(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. _ = rowSet if changed != 1 { t.Fatalf("expected 1 changed, got %v", changed) } - if false { - cur, err := tx.cursor(name) - PanicOn(err) - - cur.dump() - - _ = cur.tx.dumpAllPages(true) - } } func TestCursor_RoaringImport_clear_bits(t *testing.T) { @@ -93,7 +85,7 @@ func TestCursor_RoaringImport_clear_bits(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) _ = rowSet if changed != 1 { @@ -104,21 +96,12 @@ func TestCursor_RoaringImport_clear_bits(t *testing.T) { clear = true itr2 := getRoaringIter([]uint64{1}...) - changed, rowSet, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil) + changed, rowSet, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize) PanicOn(err) _ = rowSet if changed != 1 { t.Fatalf("expected 1 changed on clear true, got %v", changed) } - - if false { - cur, err := tx.cursor(name) - PanicOn(err) - - cur.dump() - - _ = cur.tx.dumpAllPages(true) - } } func TestCursor_RoaringImport_two_leaves(t *testing.T) { @@ -152,20 +135,12 @@ func TestCursor_RoaringImport_two_leaves(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. _ = rowSet if changed != 6000 { t.Fatalf("expected 6000 bits changed, got %v", changed) } - if false { - cur, err := tx.cursor(name) - PanicOn(err) - - cur.dump() - - _ = cur.tx.dumpAllPages(true) - } } func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) { @@ -211,7 +186,7 @@ func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) { defer tx.Rollback() //vv("DONE WITH Add()") - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != expectedBitsChanged-3000 { t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged-3000, changed) @@ -219,37 +194,23 @@ func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) { //vv("changed on set is %v", changed) //vv("about to do itr2, that starts with key %v", itr2.ContainerKeys()[0]) - changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil) + changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize) PanicOn(err) if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) } - //vv("done with itr2") - - dump := func() { - cur, err := tx.cursor(name) - PanicOn(err) - //cur.dump() - _ = cur.tx.dumpAllPages(true) - } - _ = dump - //dump() - //vv("now clear") - // now clear clear = true //itr3 := getRoaringIter(want[len(want)-3000:]...) itr3 := getRoaringIter(want...) - changed, _, err = tx.ImportRoaringBits(name, itr3, clear, false, rowSize, nil) + changed, _, err = tx.ImportRoaringBits(name, itr3, clear, false, rowSize) PanicOn(err) if changed != expectedBitsChanged { // cursor_internal_test.go:235: expected 2,724,000 bits changed, got 2,721,000 t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed) } - - //dump() } func TestCursor_RoaringImport_auto_many_leaves(t *testing.T) { @@ -292,34 +253,21 @@ func TestCursor_RoaringImport_auto_many_leaves(t *testing.T) { defer tx.Rollback() //vv("DONE WITH Add()") - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != expectedBitsChanged { t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed) } - //vv("changed on set is %v", changed) - - dump := func() { - cur, err := tx.cursor(name) - PanicOn(err) - //cur.dump() - _ = cur.tx.dumpAllPages(true) - } - _ = dump - //dump() - //vv("now clear") // now clear clear = true itr2 := getRoaringIter(want...) - changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil) + changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize) PanicOn(err) if changed != expectedBitsChanged { t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed) } - - //dump() } func TestCursor_putBranchCellsHandlesLotsOfNewBranchesAtTheRoot(t *testing.T) { @@ -430,8 +378,6 @@ func TestCursor_putBranchCellsHandlesLotsOfNewBranchesAtTheRoot(t *testing.T) { err = c.putBranchCells(0, branches) PanicOn(err) - - //c.tx.dumpAllPages(true) } func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) { @@ -475,29 +421,18 @@ func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - dump := func() { - cur, err := tx.cursor(name) - PanicOn(err) - //cur.dump() - _ = cur.tx.dumpAllPages(true) - } - _ = dump for i := 0; i < NbranchCells; i++ { itr := getRoaringIter(want[i*3000 : (i+1)*3000]...) - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) } - //vv("changed on set is %v", changed) - //dump() } - //vv("now clear") - // now clear clear = true @@ -505,13 +440,11 @@ func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) { itr := getRoaringIter(want[i*3000 : (i+1)*3000]...) - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) } - //vv("changed on set is %v", changed) - //dump() } } @@ -558,35 +491,22 @@ func TestCursor_from_B_to_C(t *testing.T) { tx := MustBegin(t, db, true) defer tx.Rollback() - dump := func() { - cur, err := tx.cursor(name) - PanicOn(err) - //cur.dump() - _ = cur.tx.dumpAllPages(true) - } - _ = dump itr := getRoaringIter(want[:6000]...) - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 6000 { t.Fatalf("expected %v bits changed, got %v", 6000, changed) } - //vv("changed on set is %v", changed) - //dump() //vv("STARTING TO ADD C") itr = getRoaringIter(want[6000:9000]...) - changed, _, err = tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err = tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) } - //vv("changed on set is %v", changed) - //dump() - - //vv("now clear") // now clear clear = true @@ -595,12 +515,10 @@ func TestCursor_from_B_to_C(t *testing.T) { itr := getRoaringIter(want[i*3000 : (i+1)*3000]...) - changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil) + changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize) PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db. if changed != 3000 { t.Fatalf("expected %v bits changed, got %v", 3000, changed) // failing here got 0 } - //vv("changed on clear is %v", changed) - //dump() } } diff --git a/rbf/cursorx.go b/rbf/cursorx.go index 0916d2c36..9b2a61111 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -174,7 +174,7 @@ func intoContainer(l leafCell, tx *Tx, replacing *roaring.Container, target []by orig := l.Data var cpMaybe []byte var mapped bool - if storage.EnableRowCache() || tx.db.cfg.DoAllocZero { + if storage.RowCacheEnabled() || tx.db.cfg.DoAllocZero { // make a copy, otherwise the rowCache will see corrupted data // or mmapped data that may disappear. cpMaybe = target[:len(orig)] @@ -191,7 +191,7 @@ func intoContainer(l leafCell, tx *Tx, replacing *roaring.Container, target []by case ContainerTypeBitmapPtr: _, bm, _ := tx.leafCellBitmap(toPgno(cpMaybe)) cloneMaybe := bm - if storage.EnableRowCache() { + if storage.RowCacheEnabled() { cloneMaybe = (*[1024]uint64)(unsafe.Pointer(&target[0]))[:1024] copy(cloneMaybe, bm) } @@ -217,7 +217,7 @@ func toContainer(l leafCell, tx *Tx) (c *roaring.Container) { orig := l.Data var cpMaybe []byte var mapped bool - if storage.EnableRowCache() || tx.db.cfg.DoAllocZero { + if storage.RowCacheEnabled() || tx.db.cfg.DoAllocZero { // make a copy, otherwise the rowCache will see corrupted data // or mmapped data that may disappear. cpMaybe = make([]byte, len(orig)) @@ -234,7 +234,7 @@ func toContainer(l leafCell, tx *Tx) (c *roaring.Container) { case ContainerTypeBitmapPtr: _, bm, _ := tx.leafCellBitmap(toPgno(cpMaybe)) cloneMaybe := bm - if storage.EnableRowCache() { + if storage.RowCacheEnabled() { cloneMaybe = make([]uint64, len(bm)) copy(cloneMaybe, bm) } diff --git a/rbf/db.go b/rbf/db.go index f1623ded8..4f88de259 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -319,7 +319,7 @@ func (db *DB) Close() (err error) { // least a single hot bit inside the db // in order to return hasAnyRecords true. // -// HasData is used by backend migration and blue/green checks. +// HasData is used by backend migration. // // If there is a disk error we return (false, error), so always // check the error before deciding if hasAnyRecords is valid. diff --git a/rbf/tx.go b/rbf/tx.go index f126cfbf9..9ecbe9a02 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -23,7 +23,6 @@ import ( "sync" "github.com/benbjohnson/immutable" - "github.com/molecula/featurebase/v2/hash" "github.com/molecula/featurebase/v2/roaring" txkey "github.com/molecula/featurebase/v2/short_txkey" . "github.com/molecula/featurebase/v2/vprint" @@ -1299,27 +1298,6 @@ func (tx *Tx) Min(name string) (uint64, bool, error) { return uint64((cell.Key << 16) | uint64(cell.firstValue(tx))), true, nil } -func (tx *Tx) UnionInPlace(name string, others ...*roaring.Bitmap) error { - rbm, err := tx.RoaringBitmap(name) - PanicOn(err) - - rbm.UnionInPlace(others...) - // iterate over the containers that changed within rbm, and write them back to disk. - - it, found := rbm.Containers.Iterator(0) - _ = found // don't care about the value of found, because first containerKey might be > 0 - - for it.Next() { - containerKey, rc := it.Value() - - // TODO: only write the changed ones back, as optimization? - // Compare to ImportRoaringBits. - err := tx.PutContainer(name, containerKey, rc) - PanicOn(err) - } - return nil -} - // roaring.countRange counts the number of bits set between [start, end). func (tx *Tx) CountRange(name string, start, end uint64) (uint64, error) { tx.mu.RLock() @@ -1538,130 +1516,7 @@ func (si *emptyContainerIterator) Value() (uint64, *roaring.Container) { return 0, nil } -func (tx *Tx) Dump(short bool, shard uint64) { - fmt.Println(tx.DumpString(short, shard)) -} -func (tx *Tx) DumpString(short bool, shard uint64) (r string) { - - r = "allkeys:[\n" - - // grab root records, for a list of bitmaps. - records, err := tx.RootRecords() - PanicOn(err) - n := 0 - - for itr := records.Iterator(); !itr.Done(); { - name, _ := itr.Next() - - c, err := tx.cursor(name.(string)) - PanicOn(err) - defer c.Close() - - err = c.First() // First will rewind to beginning. - if err == io.EOF { - r += "" - n++ - continue - } - PanicOn(err) - for { - err := c.Next() - if err == io.EOF { - break - } - PanicOn(err) - - elem := &c.stack.elems[c.stack.top] - leafPage, _, err := c.tx.readPage(elem.pgno) - PanicOn(err) - cell := readLeafCell(leafPage, elem.index) - - ckey := cell.Key - ct := toContainer(cell, tx) - - s := stringOfCkeyCt(ckey, ct, name.(string), short, true) - r += s - n++ - } - } - if n == 0 { - return "" - } - // note that we can have a bitmap present, but it can be empty - r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + "\n" - - return "rbf-" + r -} - -func containerToBytes(ct *roaring.Container) []byte { - - ty := roaring.ContainerType(ct) - switch ty { - case roaring.ContainerNil: - PanicOn("nil container") - case roaring.ContainerArray: - return fromArray16(roaring.AsArray(ct)) - case roaring.ContainerBitmap: - return fromArray64(roaring.AsBitmap(ct)) - case roaring.ContainerRun: - return fromInterval16(roaring.AsRuns(ct)) - } - PanicOn(fmt.Sprintf("unknown container type '%v'", int(ty))) - return nil -} - -func bitmapAsString(rbm *roaring.Bitmap) (r string) { - r = "c(" - slc := rbm.Slice() - width := 0 - s := "" - for _, v := range slc { - if width == 0 { - s = fmt.Sprintf("%v", v) - } else { - s = fmt.Sprintf(", %v", v) - } - width += len(s) - r += s - if width > 70 { - r += ",\n" - width = 0 - } - } - if width == 0 && len(r) > 2 { - r = r[:len(r)-2] - } - return r + ")" -} - -func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string, short, showHash bool) (s string) { - - hsh := "" - if showHash { - by := containerToBytes(ct) - hsh = hash.Blake3sum16(by) - } - - cts := roaring.NewSliceContainers() - cts.Put(ckey, ct) - rbm := &roaring.Bitmap{Containers: cts} - srbm := bitmapAsString(rbm) - - var pre string - if len(rrName) > 0 { - pre = txkey.PrefixToString([]byte(rrName)) - } - bkey := pre + fmt.Sprintf("ckey@%020d", ckey) - - s = fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hsh, ct.N()) - - if !short { - s += " ......." + srbm + "\n" - } - return -} - -func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { +func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { // begin write boilerplate if tx.db == nil { diff --git a/rbf/tx_test.go b/rbf/tx_test.go index 771e17722..f7cd78d6a 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -16,14 +16,12 @@ package rbf_test import ( "fmt" - "math" "math/rand" "sync" "testing" "time" "github.com/molecula/featurebase/v2/rbf" - txkey "github.com/molecula/featurebase/v2/short_txkey" ) func TestTx_CommitRollback(t *testing.T) { @@ -496,29 +494,6 @@ func BenchmarkTx_Contains(b *testing.B) { } } -func TestTx_Dump(t *testing.T) { - db := MustOpenDB(t) - defer MustCloseDB(t, db) - tx := MustBegin(t, db, true) - defer tx.Rollback() - - index, field, view, shard := "i", "f", "v", uint64(15) - - nm := rbfName(index, field, view, shard) - - if err := tx.CreateBitmap(nm); err != nil { - t.Fatal(err) - } else if _, err := tx.Add(nm, 0x00000001, 0x00000002, 0x00010003, 0x00030004); err != nil { - t.Fatal(err) - } - - // test that we don't crash, and get *something* back - s := tx.DumpString(true, math.MaxUint64) - if s == "" { - panic("should have had 3 containers!") - } -} - func TestTx_CreateBitmap(t *testing.T) { t.Run("Bulk", func(t *testing.T) { db := MustOpenDB(t) @@ -542,7 +517,3 @@ func TestTx_CreateBitmap(t *testing.T) { } }) } - -func rbfName(index, field, view string, shard uint64) string { - return string(txkey.Prefix(index, field, view, shard)) -} diff --git a/rbf/util.go b/rbf/util.go index fede5c190..7a44120a7 100644 --- a/rbf/util.go +++ b/rbf/util.go @@ -15,13 +15,16 @@ package rbf import ( "fmt" - "io" "strings" txkey "github.com/molecula/featurebase/v2/short_txkey" . "github.com/molecula/featurebase/v2/vprint" ) +// we don't currently use dumpAllPages but it's tricky enough to get right +// that it's probably worth keeping as a debugging tool. +var _ = (*Tx).dumpAllPages + func (tx *Tx) dumpAllPages(showLeaves bool) error { infos, err := tx.PageInfos() @@ -213,56 +216,6 @@ func prefixToString(s string) (ret string) { return txkey.PrefixToString([]byte(s)) } -func (c *Cursor) dump() { - fmt.Printf("\n Cursor %p has bitmaps:\n%v\n", c, c.debugStringBitmaps()) -} - -var _ = (&Cursor{}).dump -var _ = (&Cursor{}).debugStringBitmaps - -func (c_orig *Cursor) debugStringBitmaps() (r string) { - - // work with a totally new Cursor, so we don't impact our current cursor - // so any test using the cursor isn't disturbed. - c2 := Cursor{tx: c_orig.tx} - c2.stack.elems[0] = c_orig.stack.elems[0] - err := c2.First() - if err != nil { - if err == io.EOF { - // ok, can be empty - return "" - } else { - panic(err) - } - } - n := 0 - for { - err := c2.Next() - if err == io.EOF { - break - } - PanicOn(err) - - //instead of cell := c2.cell() - elem := &c2.stack.elems[c2.stack.top] - leafPage, _, err := c2.tx.readPage(elem.pgno) - PanicOn(err) - cell := readLeafCell(leafPage, elem.index) - - ckey := cell.Key - ct := toContainer(cell, c2.tx) - const short = true - s := stringOfCkeyCt(ckey, ct, "", short, true) - r += s - n++ - } - - if n == 0 { - return "" - } - return -} - ///////////////// happy linter var _ = printMetaPage diff --git a/roaring/roaring.go b/roaring/roaring.go index e35487a3e..4fd8a50c0 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2434,13 +2434,13 @@ func (b *Bitmap) writeOp(op *op) error { // Iterator returns a new iterator for the bitmap. func (b *Bitmap) Iterator() *Iterator { - itr := NewIterator(&BitmapIteratorFinder{b}) + itr := &Iterator{bitmap: b} itr.Seek(0) return itr } func (b *Bitmap) IteratorAt(start uint64) *Iterator { - itr := NewIterator(&BitmapIteratorFinder{b}) + itr := &Iterator{bitmap: b} itr.Seek(start) return itr } @@ -2701,37 +2701,18 @@ type BitmapInfo struct { From, To uintptr // if set, indicates the address range used when unpacking } -type IteratorFinder interface { - FindIterator(uint64) (ContainerIterator, bool) - Close() -} -type BitmapIteratorFinder struct { - bitmap *Bitmap -} - -func (bif *BitmapIteratorFinder) FindIterator(seek uint64) (ContainerIterator, bool) { - return bif.bitmap.Containers.Iterator(seek) -} -func (bif *BitmapIteratorFinder) Close() {} - // Iterator represents an iterator over a Bitmap. type Iterator struct { - finder IteratorFinder + bitmap *Bitmap citer ContainerIterator key uint64 c *Container j, k int32 // i: container; j: array index, bit index, or run index; k: offset within the run } -// NewIterator requires f as an IteratorFinder, it will -// crash if f is nil. -func NewIterator(f IteratorFinder) *Iterator { - return &Iterator{finder: f} -} - -func (itr *Iterator) Close() { - itr.finder.Close() -} +// This exists because we used to support a backend which needed it, and I +// don't want to re-experience the joy of figuring out where close calls are needed. +func (itr *Iterator) Close() {} // Seek moves to the first value equal to or greater than `seek`. func (itr *Iterator) Seek(seek uint64) { @@ -2740,7 +2721,7 @@ func (itr *Iterator) Seek(seek uint64) { itr.k = -1 // Move to the correct container. - itr.citer, _ = itr.finder.FindIterator(highbits(seek)) + itr.citer, _ = itr.bitmap.Containers.Iterator(highbits(seek)) if !itr.citer.Next() { itr.c = nil return // eof @@ -7537,10 +7518,6 @@ func (c *Container) Difference(other *Container) *Container { return difference(c, other) } -func NewSliceContainers() *sliceContainers { - return newSliceContainers() -} - // Slice returns an array of the values in the container as uint16. // Do NOT modify the result; it could be the container's actual storage. func (c *Container) Slice() (r []uint16) { diff --git a/rrtx.go b/rrtx.go index d5504cecb..247ae77f1 100644 --- a/rrtx.go +++ b/rrtx.go @@ -15,9 +15,7 @@ package pilosa import ( - "bytes" "fmt" - "io" "os" "path/filepath" "sort" @@ -49,27 +47,10 @@ type RoaringTx struct { w *RoaringWrapper } -func (tx *RoaringTx) IsDone() (done bool) { - tx.mu.Lock() - done = tx.done - tx.mu.Unlock() - return -} - func (tx *RoaringTx) Type() string { return RoaringTxn } -func (tx *RoaringTx) Dump(short bool, shard uint64) { - o := tx.o - o.Shard = shard - fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys(short, false, o)) -} - -func (tx *RoaringTx) UseRowCache() bool { - return storage.EnableRowCache() -} - // based on view.openFragments() func roaringMapOfShards(optionalViewPath string) (shardMap map[uint64]bool, err error) { @@ -112,10 +93,6 @@ func roaringMapOfShards(optionalViewPath string) (shardMap map[uint64]bool, err return } -func (tx *RoaringTx) Pointer() string { - return fmt.Sprintf("%p", tx) -} - // NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE // the transaction Commits or Rollsback. func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { @@ -127,33 +104,16 @@ func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roa // ImportRoaringBits return values changed and rowSet will be inaccurate if // the data []byte is supplied. This mimics the traditional roaring-per-file // and should be faster. -func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { +func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { f, err := tx.getFragment(index, field, view, shard) if err != nil { return 0, nil, err } - if len(data) > 0 { - // changed and rowSet are ignored anyway when len(data) > 0; - // when we are called from fragment.fillFragmentFromArchive() - // which is the only place the data []byte is supplied. - // blueGreenTx also turns off the checks in this case. - return 0, nil, f.readStorageFromArchive(bytes.NewBuffer(data)) - } changed, rowSet, err = f.storage.ImportRoaringRawIterator(rit, clear, true, rowSize) return } -func (tx *RoaringTx) Readonly() bool { - return !tx.write -} - -func (tx *RoaringTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - frag, err := tx.getFragment(index, field, view, shard) - PanicOn(err) - frag.incrementOpN(changedN) -} - func (c *RoaringTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { return GenericApplyFilter(c, index, field, view, shard, ckey, filter) } @@ -279,15 +239,6 @@ func (tx *RoaringTx) Min(index, field, view string, shard uint64) (uint64, bool, return v, ok, nil } -func (tx *RoaringTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - b, err := tx.bitmap(index, field, view, shard) - if err != nil { - return err - } - b.UnionInPlace(others...) - return nil -} - func (tx *RoaringTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) { b, err := tx.bitmap(index, field, view, shard) if err != nil { @@ -381,33 +332,6 @@ func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.B return frag.storage, nil } -func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - file, err := os.Open(fragmentPathForRoaring) // open the fragment file - if err != nil { - return nil, -1, err - } - fi, err := file.Stat() - if err != nil { - return nil, -1, errors.Wrap(err, "statting") - } - sz = fi.Size() - r = file - return -} - -func (tx *RoaringTx) Group() *TxGroup { - return tx.o.Group -} - -func (tx *RoaringTx) Options() Txo { - return tx.o -} - -// Sn retreives the serial number of the Tx. -func (tx *RoaringTx) Sn() int64 { - return tx.sn -} - func roaringGetFieldView2Shards(idx *Index) (vs *FieldView2Shards, err error) { vs = NewFieldView2Shards() @@ -651,18 +575,12 @@ func (w *RoaringWrapper) CleanupTx(tx Tx) { return } r.done = true - - r.o.dbs.Cleanup(tx) // release the read/write lock. } func (w *RoaringWrapper) OpenListString() (r string) { return "RoaringWrapper.OpenListString() not yet implemented" } -func (w *RoaringWrapper) OpenSnList() (slc []int64) { - return nil -} - func (w *RoaringWrapper) CloseDB() error { return errors.New("CloseDB not supported in roaring") } @@ -721,21 +639,12 @@ func (w *RoaringWrapper) IsClosed() (closed bool) { return } -func (w *RoaringWrapper) DeleteDBPath(dbs *DBShard) (err error) { - //vv("RoaringWrapper.DeleteDBPath called on dbs = '%#v'", dbs) - path := dbs.pathForType(roaringTxn) - return os.RemoveAll(path) -} - func (w *RoaringWrapper) DeleteField(index, field, fieldPath string) error { //vv("RoaringWrapper.DeleteField(index = '%v', field = '%v', fieldPath = '%v'", index, field, fieldPath) // match txn sn count vs lmdb/etc. atomic.AddInt64(&globalNextTxSnRoaring, 1) - // under blue-green bolt_roaring, the directory will not be found, b/c bolt will have - // already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs: - // "If the path does not exist, RemoveAll returns nil (no error)" err := os.RemoveAll(fieldPath) if err != nil { return errors.Wrap(err, "removing directory") diff --git a/server.go b/server.go index 8912d38bb..63077302b 100644 --- a/server.go +++ b/server.go @@ -334,8 +334,8 @@ func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption { } // OptServerStorageConfig is a functional option on Server used to specify the -// transactional-storage backend to use, resulting in RoaringTx, RbfTx, -// BadgerTx, or a blueGreen* Tx being used for all Tx interface calls. +// transactional-storage backend to use, resulting in RoaringTx or RbfTx +// being used for all Tx interface calls. func OptServerStorageConfig(cfg *storage.Config) ServerOption { return func(s *Server) error { s.holderConfig.StorageConfig = cfg diff --git a/server/cluster_test.go b/server/cluster_test.go index 1f7d59b14..51b8c8ebf 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -142,19 +142,6 @@ func TestClusterResize_EmptyNodes(t *testing.T) { // Ensure that adding a node correctly resizes the cluster. func TestClusterResize_AddNode(t *testing.T) { - // Why are we skipping this test under blue-green with Roaring? - // - // We see red test: during resize during importRoaringBits - // PILOSA_STORAGE_BACKEND=rbf_roaring go test -v -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0" -run TestClusterResize_AddNode/"ContinuousShards" - // green: - // PILOSA_STORAGE_BACKEND=roaring_rbf go test -v -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0" -run TestClusterResize_AddNode/"ContinuousShards" - // - // but rbf_badger and badger_rbf are both green (use the same data values for containers). - // - // Conclude: roaring reads a different size of data []byte in (due to ops log) bits vs others (RBF, badger), so - // we can't do blue-green with roaring on this test. - skipTestUnderBlueGreenWithRoaring(t) - t.Run("NoData", func(t *testing.T) { clus := test.MustRunCluster(t, 3) defer clus.Close() @@ -344,8 +331,6 @@ func TestClusterResize_AddNode(t *testing.T) { // Ensure that adding a node correctly resizes the cluster. func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { - skipTestUnderBlueGreenWithRoaring(t) - t.Run("WithIndex", func(t *testing.T) { c := test.MustRunCluster(t, 3) defer c.Close() @@ -630,12 +615,3 @@ func TestClusterMutualTLS(t *testing.T) { t.Fatal(err) } } - -func skipTestUnderBlueGreenWithRoaring(t *testing.T) { - src := pilosa.CurrentBackend() - if strings.Contains(src, "_") { - if strings.Contains(src, "roaring") { - t.Skip("skip for roaring blue-green") - } - } -} diff --git a/server/config.go b/server/config.go index dc6ae08ae..344440ded 100644 --- a/server/config.go +++ b/server/config.go @@ -207,15 +207,8 @@ type Config struct { // Storage.Backend determines which Tx implementation the holder/Index will // use; one of the available transactional-storage engines. Choices are - // listed in the string constants below. Should be one of "roaring","bolt", - // "rbf", "bolt_roaring", "roaring_bolt", "rbf_roaring", "roaring_rbf", - // "bolt_rbf", "rbf_bolt", or any later addition. The engines with _ - // underscore indicate use of a blueGreenTx with a comparison of values back - // from each Tx method, and a panic if they differ. This is an effective - // test for consistency. If "rbf_roaring" is specified, then the roaring - // values are the ones actually returned from the blueGreenTx. If - // "roaring_rbf" is chosen, then the RBF values are the ones actually - // returned from the blueGreenTx. + // listed in the string constants below. Should be one of "roaring" or + // "rbf". Storage *storage.Config `toml:"storage"` // RowcacheOn, if true, turns on the row cache for all storage backends. diff --git a/stattx.go b/stattx.go index 9dc5ab5f9..65a87fdf3 100644 --- a/stattx.go +++ b/stattx.go @@ -16,7 +16,6 @@ package pilosa import ( "fmt" - "io" "math" "runtime" "sort" @@ -151,8 +150,7 @@ type kall int // constants for kall argument to callStats.add() const ( - kIncrementOpN kall = iota - kNewTxIterator + kNewTxIterator kall = iota kImportRoaringBits kRollback kCommit @@ -169,23 +167,14 @@ const ( kCount kMax kMin - kUnionInPlace kCountRange kOffsetRange - kRoaringBitmapReader - kSliceOfShards kLast // mark the end, always keep this last. The following aren't tracked atm: kType - kDump - kReadonly - kPointer - kUseRowCache ) func (k kall) String() string { switch k { - case kIncrementOpN: - return "kIncrementOpN" case kNewTxIterator: return "kNewTxIterator" case kImportRoaringBits: @@ -220,62 +209,21 @@ func (k kall) String() string { return "kMax" case kMin: return "kMin" - case kUnionInPlace: - return "kUnionInPlace" case kCountRange: return "kCountRange" case kOffsetRange: return "kOffsetRange" - case kRoaringBitmapReader: - return "kRoaringBitmapReader" - case kSliceOfShards: - return "kSliceOfShards" case kLast: return "kLast" case kType: return "kType" - case kDump: - return "kDump" - case kReadonly: - return "kReadonly" - case kPointer: - return "kPointer" - case kUseRowCache: - return "kUseRowCache" } PanicOn(fmt.Sprintf("unknown kall '%v'", int(k))) return "" } -var _ = newStatTx // happy linter -var _ = kPointer -var _ = kUseRowCache -var _ = kType -var _ = kDump -var _ = kReadonly - var _ Tx = (*statTx)(nil) -func (c *statTx) Group() *TxGroup { - return c.b.Group() -} - -func (c *statTx) Options() Txo { - return c.b.Options() -} - -//IncrementOpN -func (c *statTx) IncrementOpN(index, field, view string, shard uint64, changedN int) { - me := kIncrementOpN - - t0 := time.Now() - defer func() { - c.stats.add(me, time.Since(t0)) - }() - - c.b.IncrementOpN(index, field, view, shard, changedN) -} - func (c *statTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator { me := kNewTxIterator @@ -286,7 +234,7 @@ func (c *statTx) NewTxIterator(index, field, view string, shard uint64) *roaring return c.b.NewTxIterator(index, field, view, shard) } -func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) { +func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) { me := kImportRoaringBits t0 := time.Now() @@ -299,25 +247,7 @@ func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit PanicOn(r) } }() - return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data) -} - -func (c *statTx) Dump(short bool, shard uint64) { - c.b.Dump(short, shard) -} - -func (c *statTx) Readonly() bool { - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see Readonly() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.Readonly() -} - -func (tx *statTx) Pointer() string { - return fmt.Sprintf("%p", tx) + return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize) } func (c *statTx) Rollback() { @@ -421,14 +351,6 @@ func (c *statTx) RemoveContainer(index, field, view string, shard uint64, key ui return c.b.RemoveContainer(index, field, view, shard, key) } -func (c *statTx) UseRowCache() bool { - return c.b.UseRowCache() -} - -func (c *statTx) IsDone() (done bool) { - return c.b.IsDone() -} - func (c *statTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) { me := kAdd @@ -586,23 +508,6 @@ func (c *statTx) Min(index, field, view string, shard uint64) (uint64, bool, err return c.b.Min(index, field, view, shard) } -func (c *statTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error { - me := kUnionInPlace - - t0 := time.Now() - defer func() { - c.stats.add(me, time.Since(t0)) - }() - - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.UnionInPlace(index, field, view, shard, others...) -} - func (c *statTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) { me := kCountRange @@ -636,32 +541,10 @@ func (c *statTx) OffsetRange(index, field, view string, shard, offset, start, en return c.b.OffsetRange(index, field, view, shard, offset, start, end) } -func (c *statTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) { - me := kRoaringBitmapReader - - t0 := time.Now() - defer func() { - c.stats.add(me, time.Since(t0)) - }() - - defer func() { - if r := recover(); r != nil { - AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack()) - PanicOn(r) - } - }() - return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring) -} - func (c *statTx) Type() string { return c.b.Type() } -// Sn retreives the serial number of the Tx. -func (c *statTx) Sn() int64 { - return c.b.Sn() -} - func (c *statTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) { return c.b.GetSortedFieldViewList(idx, shard) } diff --git a/storage/cache.go b/storage/cache.go index 7fda5af83..fe65a90f3 100644 --- a/storage/cache.go +++ b/storage/cache.go @@ -31,6 +31,6 @@ func SetRowCacheOn(on bool) { } } -func EnableRowCache() bool { +func RowCacheEnabled() bool { return atomic.LoadInt64(&enableRowcache) == 1 } diff --git a/tx.go b/tx.go index 13fd9fded..767969671 100644 --- a/tx.go +++ b/tx.go @@ -15,8 +15,6 @@ package pilosa import ( - "io" - "github.com/molecula/featurebase/v2/roaring" txkey "github.com/molecula/featurebase/v2/short_txkey" //txkey "github.com/molecula/featurebase/v2/txkey" @@ -45,8 +43,8 @@ const writable = true // that have not been committed. type Tx interface { - // Type returns "roaring", "rbf", "bolt", "badger_roaring", or one of the other - // blue-green Tx types at the top of txfactory.go + // Type returns "roaring", "rbf", or one of the other + // Tx types at the top of txfactory.go Type() string // Rollback must be called the end of read-only transactions. Either @@ -65,32 +63,6 @@ type Tx interface { // Commit makes the updates in the Tx visible to subsequent transactions. Commit() error - // IsDone must return true if Rollback() or Commit() has already - // been called. Otherwise it must return false. This allows - // DBWrapper.CleanupTx(tx Tx) to be idempotent. - IsDone() bool - - // Readonly returns the flag this transaction was created with - // during NewTx. If the transaction is writable, it will return false. - Readonly() bool - - // UseRowCache is used by fragment.go unprotectedRow() to determine - // dynamically at runtime if RoaringTx - // are in use, which for continuity wants to continue to use the - // rowCache, or if other storage engines (RBF, Badger) are in - // use, which will mean that the bitmap data stored by the - // rowCache can disappear as it is un-mmap-ed, causing crashes. - UseRowCache() bool - - // IncrementOpN updates internal statistics with the changedN provided. - IncrementOpN(index, field, view string, shard uint64, changedN int) - - // Pointer gives us a memory address for the underlying - // transaction for debugging. - // It is public because we use it in roaring to report invalid - // container memory access outside of a transaction. - Pointer() string - // NewTxIterator returns it, a *roaring.Iterator whose it.Next() will // successively return each uint64 stored in the conceptual roaring.Bitmap // for the specified fragment. @@ -103,8 +75,7 @@ type Tx interface { // Return value 'found' is true when the ckey container was present. // ckey of 0 gives all containers (in the fragment). // - // ContainerIterator must not have side-effects. blueGreenTx will - // call it at the very beginning of commit to verify db contents. + // ContainerIterator must not have side-effects. // // citer.Close() must be called when the client is done using it. ContainerIterator(index, field, view string, shard uint64, ckey uint64) (citer roaring.ContainerIterator, found bool, err error) @@ -154,9 +125,6 @@ type Tx interface { // Min Min(index, field, view string, shard uint64) (uint64, bool, error) - // UnionInPlace - UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error - // CountRange CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) @@ -170,31 +138,9 @@ type Tx interface { // If clear is true, the bits from rit are cleared, otherwise they are set in the // specifed fragment. // - // The data argument can be nil, its ignored for RBF/BadgerTx. It is supplied to - // RoaringTx.ImportRoaringBits() in fragment.go fragment.fillFragmentFromArchive() - // to do the traditional fragment.readStorageFromArchive() which - // does some in memory field/view/fragment metadata updates. - // It makes blueGreenTx testing viable too. - // // ImportRoaringBits return values changed and rowSet may be inaccurate if // the data []byte is supplied (the RoaringTx implementation neglects this for speed). - ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) - - RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) - - // Group returns nil or the TxGroup that this Tx is a part of. - Group() *TxGroup - - // Dump is for debugging, what does this Tx see as its database? - Dump(short bool, shard uint64) - - // Options returns the options used to create this Tx. This - // can be implementd by embedding Txo, and Txo provides the - // Options() method. - Options() Txo - - // Sn retreives the serial number of the Tx. - Sn() int64 + ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) // GetSortedFieldViewList gets the set of FieldView(s) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) @@ -202,51 +148,6 @@ type Tx interface { GetFieldSizeBytes(index, field string) (uint64, error) } -// Closer is used by Finders -type Closer interface { - Close() -} - -type Dumper interface { - // Dump is for debugging, what does this Tx see as its database? - AllDump() -} - -// TxStore has operations that will create and commit multiple -// Tx on a backing store. -type TxStore interface { - - // DeleteFragment deletes all the containers in a fragment. - // - // This is not in a Tx because it will often do too many deletes for a single - // transaction, and clients would be suprised to find their Tx had already - // been commited and they are getting an error on double-Commit. - // Instead each TxStore implementation creates and commits as many - // transactions as needed. - // - // Argument frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil. - // If not nil, it must be of type *fragment. If frag is supplied, then - // index must be equal to frag.index, field equal to frag.field, view equal - // to frag.view, and shard equal to frag.shard. - // - DeleteFragment(index, field, view string, shard uint64, frag interface{}) error - - DeleteField(index, field string) error - - // Close shuts down the database. - Close() error -} - -// RawRoaringData used by ImportRoaringBits. -// must be consumable by roaring.newRoaringIterator() -type RawRoaringData struct { - data []byte -} - -func (rr *RawRoaringData) Iterator() (roaring.RoaringIterator, error) { - return roaring.NewRoaringIterator(rr.data) -} - // GenericApplyFilter implements ApplyFilter in terms of tx.ContainerIterator, // as a convenience if a Tx backend hasn't implemented this new function yet. func GenericApplyFilter(tx Tx, index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) { diff --git a/txfactory.go b/txfactory.go index 7c0d06add..e28a6fb03 100644 --- a/txfactory.go +++ b/txfactory.go @@ -16,32 +16,22 @@ package pilosa import ( "fmt" - "io" "os" "path" "path/filepath" - "runtime" "strconv" "strings" "sync" - "syscall" - "text/tabwriter" - "github.com/molecula/featurebase/v2/hash" - "github.com/molecula/featurebase/v2/roaring" - txkey "github.com/molecula/featurebase/v2/short_txkey" - "github.com/molecula/featurebase/v2/storage" "github.com/molecula/featurebase/v2/testhook" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck "github.com/pkg/errors" - "github.com/zeebo/blake3" ) // public strings that pilosa/server/config.go can reference const ( RoaringTxn string = "roaring" RBFTxn string = "rbf" - BoltTxn string = "bolt" ) // DetectMemAccessPastTx true helps us catch places in api and executor @@ -267,7 +257,7 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) { // qcx.write reflects the top executor determination // if a write will be done at the end, so we upgrade // the "local" read Tx to be writes, so that they - // don't deadlock against themselves under blue-green. + // don't deadlock against themselves. o.Write = o.Write || qcx.write // In general, we make ALL write transactions local, and never reuse them @@ -305,9 +295,8 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) { if already { return } - o.Group = qcx.Grp tx = qcx.Txf.NewTx(o) - qcx.Grp.AddTx(tx) + qcx.Grp.AddTx(tx, o) return } @@ -351,7 +340,6 @@ func (qcx *Qcx) StartAtomicWriteTx(o Txo) { // new Tx needed tx := qcx.Txf.NewTx(o) qcx.RequiredForAtomicWriteTx = &tx - o := tx.Options() qcx.RequiredTxo = &o return } @@ -374,55 +362,23 @@ func (qcx *Qcx) StartAtomicWriteTx(o Txo) { } } -func (qcx *Qcx) SetRequiredForAtomicWriteTx(tx Tx) { - if tx == nil || NilInside(tx) { - PanicOn("cannot set nil tx in SetRequiredForAtomicWriteTx") - } - qcx.mu.Lock() - qcx.RequiredForAtomicWriteTx = &tx - o := tx.Options() - qcx.RequiredTxo = &o - qcx.mu.Unlock() -} - -func (qcx *Qcx) ClearRequiredForAtomicWriteTx() { - qcx.mu.Lock() - qcx.RequiredForAtomicWriteTx = nil - qcx.RequiredTxo = nil - qcx.mu.Unlock() -} - func (qcx *Qcx) ListOpenTx() string { return qcx.Grp.String() } // TxFactory abstracts the creation of Tx interface-level -// transactions so that RBF, BoltDB, or Roaring-fragment-files, or several +// transactions so that RBF, or Roaring-fragment-files, or several // of these at once in parallel, is used as the storage and transction layer. type TxFactory struct { typeOfTx string - mu sync.Mutex - - types []txtype // blue-green split individually here + typ txtype dbsClosed bool // idemopotent CloseDB() dbPerShard *DBPerShard holder *Holder - - blueGreenReg *blueGreenRegistry - - // allow holder to activate blue-green checking only - // once we have synced both sides at start up time. - blueGreenOff bool - - isBlueGreen bool -} - -func (f *TxFactory) Types() []txtype { - return f.types } // integer types for fast switch{} @@ -432,7 +388,6 @@ const ( noneTxn txtype = 0 roaringTxn txtype = 1 // these don't really have any transactions rbfTxn txtype = 2 - boltTxn txtype = 4 ) // DirectoryName just returns a string version of the transaction type. We @@ -445,73 +400,41 @@ func (ty txtype) DirectoryName() string { return "roaring" case rbfTxn: return "rbf" - case boltTxn: - return "boltdb" } PanicOn(fmt.Sprintf("unkown txtype %v", int(ty))) return "" } func (txf *TxFactory) NeedsSnapshot() (b bool) { - for _, ty := range txf.types { - switch ty { - case roaringTxn: - b = true - return - } - } - return + return txf.typ == roaringTxn } -func MustBackendToTxtype(backend string) (types []txtype) { - var srcs []string +func MustBackendToTxtype(backend string) (typ txtype) { if strings.Contains(backend, "_") { - srcs = strings.Split(backend, "_") - if len(srcs) != 2 { - PanicOn("only two blue-green comparisons permitted") - } - } else { - srcs = append(srcs, backend) + panic("blue-green comparisons removed") } - for i, s := range srcs { - switch s { - case RoaringTxn: // "roaring" - types = append(types, roaringTxn) - case RBFTxn: // "rbf" - types = append(types, rbfTxn) - case BoltTxn: // "bolt" - types = append(types, boltTxn) - default: - PanicOn(fmt.Sprintf("unknown backend '%v'", s)) - } - if i == 1 { - if types[1] == types[0] { - PanicOn(fmt.Sprintf("cannot blue-green the same backend on both arms: '%v'", s)) - } - } + switch backend { + case RoaringTxn: // "roaring" + return roaringTxn + case RBFTxn: // "rbf" + return rbfTxn } - return + panic(fmt.Sprintf("unknown backend '%v'", backend)) } // NewTxFactory always opens an existing database. If you // want to a fresh database, os.RemoveAll on dir/name ahead of time. // We always store files in a subdir of holderDir. func NewTxFactory(backend string, holderDir string, holder *Holder) (f *TxFactory, err error) { - types := MustBackendToTxtype(backend) + typ := MustBackendToTxtype(backend) f = &TxFactory{ - types: types, + typ: typ, typeOfTx: backend, holder: holder, } - if len(types) == 2 { - f.blueGreenReg = newBlueGreenReg(types) - f.isBlueGreen = true - // blue-green can never use the rowCache. - storage.SetRowCacheOn(false) - } - f.dbPerShard = f.NewDBPerShard(types, holderDir, holder) + f.dbPerShard = f.NewDBPerShard(typ, holderDir, holder) if f.hasRBF() { holder.Logger.Infof("rbf config = %#v", holder.cfg.RBFConfig) @@ -526,15 +449,6 @@ func (f *TxFactory) Open() error { return f.dbPerShard.LoadExistingDBs() } -// UseRowCache can be more "global" than Tx at the moment, because -// we are sharing the same bool flag in rbf at the moment. If -// this changes then fragment.openStorage() will need a new way -// to determine if it should use the rowCache. Currently it -// doesn't have a tx Tx parameter, so we use the Txf instead. -func (f *TxFactory) UseRowCache() bool { - return storage.EnableRowCache() -} - // Txo holds the transaction options type Txo struct { Write bool @@ -544,23 +458,14 @@ type Txo struct { Shard uint64 dbs *DBShard - per *DBPerShard - - Group *TxGroup - - blueGreenOff bool -} - -func (o Txo) String() string { - return fmt.Sprintf("Txo{Write:%v, Index:%v Shard:%v Group:%p}", o.Write, o.Index.name, o.Shard, o.Group) } func (f *TxFactory) TxType() string { return f.typeOfTx } -func (f *TxFactory) TxTypes() []txtype { - return f.types +func (f *TxFactory) TxTyp() txtype { + return f.typ } func (f *TxFactory) DeleteIndex(name string) (err error) { @@ -577,10 +482,6 @@ func (f *TxFactory) DeleteFragmentFromStore( return f.dbPerShard.DeleteFragment(index, field, view, shard, frag) } -func (f *TxFactory) DumpAll() { - f.dbPerShard.DumpAll() -} - // IndexUsageDetails computes the sum of filesizes used by the node, broken down // by index, field, fragments and keys. func (f *TxFactory) IndexUsageDetails(isClosing func() bool) (map[string]IndexUsage, uint64, error) { @@ -769,7 +670,6 @@ func directoryUsage(fname string, recursive bool) (uint64, error) { // CloseIndex is a no-op. This seems to be in place for debugging purposes. func (f *TxFactory) CloseIndex(idx *Index) error { - //idx.Dump("CloseIndex") return nil } @@ -790,24 +690,24 @@ func init() { } } -// TxGroup holds a set of read and a set of write transactions -// that will en-mass have Rollback() (for the read set) and -// Commit() (for the write set) called on +// TxGroup holds a set of read transactions +// that will en-mass have Rollback() (for the read set) called on // them when TxGroup.Finish() is invoked. // Alternatively, TxGroup.Abort() will call Rollback() // on all Tx group memebers. +// +// It used to have writes but we never actually used that because +// of the Qcx needing to make every commit get its own transaction. type TxGroup struct { mu sync.Mutex fac *TxFactory reads []Tx - writes []Tx finished bool all map[grpkey]Tx } type grpkey struct { - write bool index string shard uint64 } @@ -822,7 +722,7 @@ func (g *TxGroup) AlreadyHaveTx(o Txo) (tx Tx, already bool) { mustHaveIndexShard(&o) g.mu.Lock() defer g.mu.Unlock() - key := grpkey{write: o.Write, index: o.Index.name, shard: o.Shard} + key := grpkey{index: o.Index.name, shard: o.Shard} tx, already = g.all[key] return } @@ -830,21 +730,14 @@ func (g *TxGroup) AlreadyHaveTx(o Txo) (tx Tx, already bool) { func (g *TxGroup) String() (r string) { g.mu.Lock() defer g.mu.Unlock() - if len(g.reads) == 0 && len(g.writes) == 0 { + if len(g.reads) == 0 { return "" } - - i := 0 r += "\n" - for _, tx := range g.reads { - r += fmt.Sprintf("[%v]read: _sn_ %v %v, \n", i, tx.Sn(), tx.Options()) - i++ + for i, tx := range g.reads { + r += fmt.Sprintf("[%v]read: %#v,\n", i, tx) } - for _, tx := range g.writes { - r += fmt.Sprintf("[%v]write: _sn_ %v %v, \n", i, tx.Sn(), tx.Options()) - i++ - } - return + return r } // NewTxGroup @@ -857,7 +750,7 @@ func (f *TxFactory) NewTxGroup() (g *TxGroup) { } // AddTx adds tx to the group. -func (g *TxGroup) AddTx(tx Tx) { +func (g *TxGroup) AddTx(tx Tx, o Txo) { g.mu.Lock() defer g.mu.Unlock() if g.finished { @@ -867,15 +760,9 @@ func (g *TxGroup) AddTx(tx Tx) { PanicOn("Cannot add nil Tx to TxGroup") } - if tx.Readonly() { - g.reads = append(g.reads, tx) - } else { - g.writes = append(g.writes, tx) - } - o := tx.Options() - mustHaveIndexShard(&o) + g.reads = append(g.reads, tx) - key := grpkey{write: o.Write, index: o.Index.name, shard: o.Shard} + key := grpkey{index: o.Index.name, shard: o.Shard} prior, ok := g.all[key] if ok { PanicOn(fmt.Sprintf("already have Tx in group for this, we should have re-used it! prior is '%v'; tx='%v'", prior, tx)) @@ -893,15 +780,6 @@ func (g *TxGroup) FinishGroup() (err error) { PanicOn("in TxGroup.Finish(): TxGroup already finished") } g.finished = true - for i, tx := range g.writes { - _ = i - err0 := tx.Commit() - if err0 != nil { - if err == nil { - err = err0 // keep the first error, but Commit them all. - } - } - } for _, r := range g.reads { r.Rollback() } @@ -923,9 +801,6 @@ func (g *TxGroup) AbortGroup() { for _, r := range g.reads { r.Rollback() } - for _, tx := range g.writes { - tx.Rollback() - } } func (f *TxFactory) NewTx(o Txo) (txn Tx) { @@ -935,12 +810,6 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) { } }() - if f.isBlueGreen { - f.mu.Lock() - o.blueGreenOff = f.blueGreenOff - f.mu.Unlock() - } - indexName := "" if o.Index != nil { indexName = o.Index.name @@ -963,10 +832,8 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) { if dbs.Shard != o.Shard { PanicOn(fmt.Sprintf("asked for o.Shard=%v but got dbs.Shard=%v", int(o.Shard), int(dbs.Shard))) } - //vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.types='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.types, dbs.W) - - o.dbs = dbs // our specific database per shard. - o.per = f.dbPerShard // for top level debug Dumps + //vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.typ='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.typ, dbs.W) + o.dbs = dbs tx, err := dbs.NewTx(o.Write, indexName, o) if err != nil { @@ -984,8 +851,6 @@ func (ty txtype) String() string { return "roaring" case rbfTxn: return "rbf" - case boltTxn: - return "bolt" } PanicOn(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty))) return "" @@ -1022,147 +887,6 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64, return } -// hashOnly means only show the value hash, not the content bits. -// showOps means display the ops log. -func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool, o Txo) (r string) { - paths, err := listFilesUnderDir(idx.path, false, "", true) - PanicOn(err) - index := idx.name - - r = "allkeys:[\n" - n := 0 - for _, relpath := range paths { - field, view, shard, err := fragmentSpecFromRoaringPath(relpath) - if err != nil { - continue // ignore .meta paths - } - if shard != o.Shard { - continue // only print the shard the Txo is on. - } - abspath := idx.path + sep + relpath - - s, _, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard, showOps, hashOnly, os.Stdout) - PanicOn(err) - //r += fmt.Sprintf("path:'%v' fragment contains:\n") + s - //if s == "" { - //s = "" - //} - r += s - n++ - } - if n == 0 { - return "" - } - // note that we can have a bitmap present, but it can be empty - r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + "\n" - - return "roaring-" + r -} - -func RoaringFragmentChecksum(path string, index, field, view string, shard uint64) (r string, hotbits int) { - defer func() { - r := recover() - if r != nil { - PanicOn(fmt.Sprintf("caught PanicOn on path='%v', index='%v', field='%v', view='%v', shard='%v': %v", - path, index, field, view, shard, r)) - } - }() - hasher := blake3.New() - showOps := false - hashOnly := true - hash, hotbits, err := stringifiedRawRoaringFragment(path, index, field, view, shard, showOps, hashOnly, hasher) - PanicOn(err) - fmt.Fprintf(hasher, "%v/%v/%v/%v/%v", index, field, view, shard, hash) - var buf [16]byte - _, _ = hasher.Digest().Read(buf[0:]) - return fmt.Sprintf("%x", buf), hotbits - -} - -func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64, showOps, hashOnly bool, w io.Writer) (r string, hotbits int, err error) { - - var info roaring.BitmapInfo - _ = info - var f *os.File - f, err = os.Open(path) - PanicOn(err) - if err != nil { - return - } - - var fi os.FileInfo - fi, err = f.Stat() - PanicOn(err) - if err != nil { - return - } - // Memory map the file. - data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED) - if err != nil { - err = errors.Wrap(err, "mmapping") - return - } - defer func() { - err := syscall.Munmap(data) - if err != nil { - PanicOn(fmt.Errorf("loadRawRoaringContainer: munmap failed: %v", err)) - } - PanicOn(f.Close()) - }() - - // Attach the mmap file to the bitmap. - var rbm *roaring.Bitmap - rbm, _, err = roaring.InspectBinary(data, true, &info) - if err != nil { - err = errors.Wrap(err, "inspecting") - return - } - - //cmd.DisplayInfo(info) - // inlined - if showOps { - pC := pointerContext{ - from: info.From, - to: info.To, - } - if info.ContainerCount > 0 { - printContainers(w, info, pC) - } - if info.Ops > 0 { - printOps(w, info) - } - } - - citer, found := rbm.Containers.Iterator(0) - _ = found // probably gonna use just the Ops log instead, so don't PanicOn if !found. - - for citer.Next() { - ckey, ct := citer.Value() - by := containerToBytes(ct) - hash := hash.Blake3sum16(by) - - cts := roaring.NewSliceContainers() - cts.Put(ckey, ct) - rbm := &roaring.Bitmap{Containers: cts} - - var srbm string - if !hashOnly { - srbm = BitmapAsString(rbm) - } - - bkey := txkey.ToString(txkey.Key(index, field, view, shard, ckey)) - - n := ct.N() - hotbits += int(n) - r += fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, n) - if !hashOnly { - r += " ......." + srbm + "\n" - } - } - - return -} - // listFilesUnderDir returns the paths of files found under directory root. // If includeRoot is true, it returns the full path, otherwise paths are relative to root. // If requriedSuffix is supplied, the returned file paths will end in that, @@ -1218,130 +942,6 @@ func fileSize(name string) (int64, error) { return fi.Size(), nil } -var _ = fileSize // happy linter - -func containerToBytes(ct *roaring.Container) []byte { - ty := roaring.ContainerType(ct) - switch ty { - case roaring.ContainerNil: - PanicOn("nil roaring.Container") - case roaring.ContainerArray: - return fromArray16(roaring.AsArray(ct)) - case roaring.ContainerBitmap: - return fromArray64(roaring.AsBitmap(ct)) - case roaring.ContainerRun: - return fromInterval16(roaring.AsRuns(ct)) - } - PanicOn(fmt.Sprintf("unknown roaring.Container type '%v'", int(ty))) - return nil -} - -type pointerContext struct { - from, to uintptr -} - -func printOps(w io.Writer, info roaring.BitmapInfo) { - fmt.Fprintln(w, " Ops:") - tw := tabwriter.NewWriter(w, 0, 8, 0, '\t', 0) - fmt.Fprintf(tw, " \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE") - printed := 0 - for _, op := range info.OpDetails { - fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size) - printed++ - } - tw.Flush() -} - -func (p *pointerContext) pretty(c roaring.ContainerInfo) string { - var pointer string - if c.Mapped { - if c.Pointer >= p.from && c.Pointer < p.to { - pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from) - } else { - pointer = fmt.Sprintf("!0x%x!", c.Pointer) - } - } else { - pointer = fmt.Sprintf("0x%x", c.Pointer) - } - return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer) -} - -// stolen from ctl/inspect.go -func printContainers(w io.Writer, info roaring.BitmapInfo, pC pointerContext) { - fmt.Fprintln(w, " Containers:") - tw := tabwriter.NewWriter(w, 0, 8, 0, '\t', 0) - fmt.Fprintf(tw, " \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n") - fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS") - c1s := info.Containers - c2s := info.OpContainers - l1 := len(c1s) - l2 := len(c2s) - i1 := 0 - i2 := 0 - var c1, c2 roaring.ContainerInfo - c1.Key = ^uint64(0) - c2.Key = ^uint64(0) - c1e := false - c2e := false - if i1 < l1 { - c1 = c1s[i1] - i1++ - c1e = true - } - if i2 < l2 { - c2 = c2s[i2] - i2++ - c2e = true - } - printed := 0 - for c1e || c2e { - c1used := false - c2used := false - var key uint64 - c1fmt := "-\t\t\t" - c2fmt := "-\t\t\t" - // If c2 exists, we'll always prefer its flags, - // if it doesn't, this gets overwritten. - flags := c2.Flags - if !c2e || (c1e && c1.Key < c2.Key) { - c1fmt = pC.pretty(c1) - key = c1.Key - c1used = true - flags = c1.Flags - } else if !c1e || (c2e && c2.Key < c1.Key) { - c2fmt = pC.pretty(c2) - key = c2.Key - c2used = true - } else { - // c1e and c2e both set, and neither key is < the other. - c1fmt = pC.pretty(c1) - c2fmt = pC.pretty(c2) - key = c1.Key - c1used = true - c2used = true - } - if c1used { - if i1 < l1 { - c1 = c1s[i1] - i1++ - } else { - c1e = false - } - } - if c2used { - if i2 < l2 { - c2 = c2s[i2] - i2++ - } else { - c2e = false - } - } - fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags) - printed++ - } - tw.Flush() -} - var _ = anyGlobalDBWrappersStillOpen // happy linter func anyGlobalDBWrappersStillOpen() bool { @@ -1351,225 +951,19 @@ func anyGlobalDBWrappersStillOpen() bool { if globalRbfDBReg.Size() != 0 { return true } - if globalBoltReg.Size() != 0 { - return true - } return false } -func (f *TxFactory) blueGreenOnIfRunningBlueGreen() { - if len(f.types) == 2 { - f.blueGreenOff = false - } -} - -func (f *TxFactory) blueGreenOffIfRunningBlueGreen() { - if len(f.types) == 2 { - f.blueGreenOff = true - } -} - func (f *TxFactory) hasRoaring() bool { - return f.types[0] == roaringTxn || (len(f.types) > 1 && f.types[1] == roaringTxn) + return f.typ == roaringTxn } func (f *TxFactory) hasRBF() bool { - return f.types[0] == rbfTxn || (len(f.types) > 1 && f.types[1] == rbfTxn) + return f.typ == rbfTxn } var _ = (&TxFactory{}).hasRoaring // happy linter -func (f *TxFactory) blueHasData() (hasData bool, err error) { - if len(f.types) != 2 { - return false, nil - } - return f.dbPerShard.HasData(0) -} - -func (f *TxFactory) greenHasData() (hasData bool, err error) { - n := len(f.types) - switch n { - case 1: - return f.dbPerShard.HasData(0) - case 2: - return f.dbPerShard.HasData(1) - } - err = fmt.Errorf("unsupported len(f.types): %v; must be 1 or 2", n) - PanicOn(err) - return -} - -// green2blue is called at the very end of Holder.Open(), so -// we know that the holder is ready to go, knowing its holder.Indexes(), fields, -// view, shards, and other metadata if any. -// -// Called by test Test_TxFactory_UpdateBlueFromGreen_OnStartup() in -// txfactory_internal_test.go as well. -// -// This is a noop if we aren't running under a blue_green PILOSA_STORAGE_BACKEND. -func (f *TxFactory) green2blue(holder *Holder) (err0 error) { - - // Holder.Open will always call us, even without blue_green. Which is fine. - // We are just a no-op in that case. - if len(f.types) != 2 { - return nil - } - - holder.Logger.Infof("green2blue analysis begins.") - - blueDest := f.types[0] - greenSrc := f.types[1] - - if blueDest == roaringTxn { - return fmt.Errorf("error: cannot migrate to 'roaring': not implemented") - } - - idxs := holder.Indexes() - - verifyInsteadOfCopy := false - - blueHasData, err := f.blueHasData() - if err != nil { - return errors.Wrap(err, "TxFactory.green2blue f.blueHasData()") - } - - greenHasData, err := f.greenHasData() - if err != nil { - return errors.Wrap(err, "TxFactory.green2blue f.greenHasData()") - } - if !blueHasData && !greenHasData { - holder.Logger.Infof("no data in blue or green. No migration or verification to do") - return nil - } - // INVAR: blue has data. - if !greenHasData { - holder.Logger.Errorf("cannot migrate from green '%v' because it has no data in it", greenSrc) - return fmt.Errorf("error: cannot migrate from green '%v' because it has no data in it", greenSrc) - } - - nGoro := runtime.NumCPU() - if nGoro < 5 { - // try to get some overlapped IO - nGoro = 5 - } - pj := newParallelJobs(nGoro) - - action := "verify" - if blueHasData { - verifyInsteadOfCopy = true - defer holder.Logger.Infof("bitmap-backend verification done : %v compared to %v", blueDest, greenSrc) - } else { - action = "migrate" - holder.Logger.Infof("bitmap-backend migration starting: populating %v from %v with %v threads", blueDest, greenSrc, nGoro) - defer holder.Logger.Infof("bitmap-backend migration done : populated %v from %v", blueDest, greenSrc) - } - firstPjobStarted := false - -indexloop: - for k, idx := range idxs { - - // scan directories - blueShards, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(blueDest, idx, "", false) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v') error fetching blueShards", idx.name)) - } - - // scan directories - greenShards, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(greenSrc, idx, "", true) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v') error fetching greenShards", idx.name)) - } - - if verifyInsteadOfCopy { - diff := f.shardSetDiff(blueShards, greenShards) - if diff != "" { - return fmt.Errorf("verifyInsteadOfCopy true, blue[%v]=%#v and green[%v]=%#v have different shards for index '%v': '%v'; stack=\n%v", blueDest, blueShards, greenSrc, greenShards, idx.name, diff, Stack()) - } - - // can also check against meta data - shards := idx.AvailableShards(localOnly).Slice() - meta := make(map[uint64]bool) - for _, shard := range shards { - meta[shard] = true - } - diff2 := f.shardSetDiff(greenShards, meta) - if diff2 != "" { - return fmt.Errorf("green[%v] = '%#v' and meta data '%#v' have different shards for index '%v': %v", greenSrc, greenShards, shards, idx.name, diff2) - } - } - - shardNum := 0 - for shard := range greenShards { - shardNum++ - shnum := shardNum - idx := idx - shard := shard - k := k - fun := func(worker int) error { - - dbs, err := f.dbPerShard.GetDBShard(idx.name, shard, idx) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v', shard='%v')", idx.name, int(shard))) - } - - holder.Logger.Infof("%v progress on index '%v' (%v of %v): on shard '%v' (%v of %v) [worker %v]", - action, idx.name, k+1, len(idxs), shard, shnum, len(greenShards), worker) - - if verifyInsteadOfCopy { - // verify all containers - err = dbs.verifyBlueEqualsGreen() - if err != nil { - return errors.Wrap(err, - fmt.Sprintf("dbs.verifyBlueEqualsGreen(blue='%v', "+ - "green='%v') for index='%v', shard='%v'", - blueDest, greenSrc, idx.name, int(shard))) - } - } else { - // the main copy work - err = dbs.populateBlueFromGreen() - if err != nil { - return errors.Wrap(err, - fmt.Sprintf("dbs.copyGreenToBlue(blue='%v', "+ - "green='%v') for index='%v', shard='%v'", - blueDest, greenSrc, idx.name, int(shard))) - } - } - return nil - } // end of fun definition - - if !pj.run(fun) { - break indexloop - } - if !firstPjobStarted { - firstPjobStarted = true - defer func() { - err1 := pj.waitForFinish() - if err0 == nil { - err0 = err1 - } - }() - } - } - } - return nil -} - -func (f *TxFactory) shardSetDiff(blueShards, greenShards map[uint64]bool) (diff string) { - nb := len(blueShards) - ng := len(greenShards) - if nb != ng { - diff = fmt.Sprintf("blueShard[%v] count = %v; greenShard[%v] count = %v; ", f.types[0], nb, f.types[1], ng) - } - bmg := mapDiff(blueShards, greenShards) // get blue - green - gmb := mapDiff(greenShards, blueShards) // get green - blue - - if len(bmg) == 0 && len(gmb) == 0 { - return "" - } - diff += fmt.Sprintf("shard diff: blueMinusGreen shards: '%#v'; greenMinusBlue shards: '%#v'", bmg, gmb) - return -} - func (f *TxFactory) GetDBShardPath(index string, shard uint64, idx *Index, ty txtype, write bool) (shardPath string, err error) { dbs, err := f.dbPerShard.GetDBShard(index, shard, idx) if err != nil { diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go index b310dbdfe..9cfa9badb 100644 --- a/txfactory_internal_test.go +++ b/txfactory_internal_test.go @@ -15,393 +15,14 @@ package pilosa import ( - "context" - "fmt" - "os" "testing" - "time" - - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck ) -func Test_TxFactory_Qcx_query_context(t *testing.T) { - src := CurrentBackend() - if src == "rbf" || src == "bolt" { - // ok - } else { - t.Skip("this test only for rbf and bolt") - } - - shard := uint64(0) - f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, shard, "") - defer f.Clean(t) - tx.Rollback() - - barrier := NewBarrier() - defer barrier.Close() - - done := make(chan bool) - - setter := func(k int) { - for i := 0; ; i++ { - barrier.WaitAtGate(0) - select { - case <-done: - return - default: - } - // add to the group txn on the txf. - qcx := idx.holder.txf.NewQcx() - - tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: idx, Shard: f.shard}) - PanicOn(err) - - // Set bits on the fragment. - if _, err := f.setBit(tx, 120, 1); err != nil { - panic(err) - } else if _, err := f.setBit(tx, 120, 6); err != nil { - panic(err) - } else if _, err := f.setBit(tx, 121, 0); err != nil { - panic(err) - } - // should have two containers set in the fragment. - - // Verify counts on rows. - if n := f.mustRow(tx, 120).Count(); n != 2 { - panic(fmt.Sprintf("unexpected count: %d", n)) - } else if n := f.mustRow(tx, 121).Count(); n != 1 { - panic(fmt.Sprintf("unexpected count: %d", n)) - } - finisher(nil) // hit the write tx.Commit path - // commit the change, and verify it is still there - PanicOn(qcx.Finish()) - qcx.Reset() - - tx, finread, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard}) - PanicOn(err) - if n := f.mustRow(tx, 120).Count(); n != 2 { - panic(fmt.Sprintf("unexpected count (reopen): %d", n)) - } else if n := f.mustRow(tx, 121).Count(); n != 1 { - panic(fmt.Sprintf("unexpected count (reopen): %d", n)) - } - finread(nil) // no-op on reads that are in a group, so must qcx.Abort() to stop them. - qcx.Abort() - qcx.Reset() - } - } - N := 1000 - for i := 0; i < N; i++ { - go setter(i) - } - time.Sleep(time.Second * 1) - close(done) - - // allow all goro to finish before Closing the lmdb.env, otherwise - // we will crash as the goroutines making Tx will try to use the env - // after it is closed. It can take quite a while. - // one writer might be blocking the other... so ask for only N-2 at first - // to avoid deadlock. - barrier.BlockUntil(N - 2) - barrier.UnblockReaders() - time.Sleep(1 * time.Second) -} - -// test TxFactory.green2blue -// -// blue_green starting with an empty or full blue database -// should copy all of green (if blue is empty); or if blue is ull, -// verify that blue has all the same bits as green. -// -// Benefits: a) we start with known identical state so our testing/comparisons can be valid; -// and b) we have an easy migration mechanism, to go from one storage format to another. -// -func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) { - checked := []string{"roaring", "rbf"} - - expectError := false - for _, blue := range checked { - for _, green := range checked { - if blue == green { - continue - } - if blue == "roaring" { - // not supported - expectError = true - } else { - expectError = false - } - blue_green := blue + "_" + green - //vv("setting blue_green to '%v'", blue_green) - - // ============================= - // Begin setup. - // - // Setup happens with green only. - - h, path, err := makeHolder(t, green) - if err != nil { - t.Fatalf("creating holder: %v", err) - } - defer os.RemoveAll(path) - //vv("path = %v", path) - - // we will manually h.Close() below - - // Write bits to separate indexes. - testSetBit(t, h, "i0", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 12345678) - - testOp := testHolderOperator{} - ctx := context.Background() - err = h.Process(ctx, &testOp) - if err != nil { - t.Fatalf("processing holder: %v", err) - } - expected := testHolderOperator{ - indexSeen: 2, indexProcessed: 2, - fieldSeen: 2, fieldProcessed: 2, - viewSeen: 2, viewProcessed: 2, - fragmentSeen: 3, fragmentProcessed: 3, - } - if testOp != expected { - t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp) - } - - // verify data is there - rowID := uint64(100) - colID := uint64(200) - _, _ = rowID, colID - testMustHaveBit(t, h, "i0", "f", rowID, colID) - testMustHaveBit(t, h, "i1", "f", 100, 200) - testMustHaveBit(t, h, "i1", "f", 100, 12345678) - - //vv("about to reopen; blue_green = '%v' but PILOSA_STORAGE_BACKEND='%v'", blue_green, os.Getenv("PILOSA_STORAGE_BACKEND")) - //h.DumpAllShards() - - //vv("after dump, about to close") - h.Close() - - //vv("after close, about to re-open") - - // can we re.Open the same holder h? hopefully without a problem. - PanicOn(h.Open()) - - //vv("h.Open() re-open worked; blue_green = '%v'; dump; with PILOSA_STORAGE_BACKEND='%v'", blue_green, os.Getenv("PILOSA_STORAGE_BACKEND")) - //h.DumpAllShards() - - testMustHaveBit(t, h, "i0", "f", rowID, colID) // panic here, colID 200 bit was cold. - testMustHaveBit(t, h, "i1", "f", 100, 200) - testMustHaveBit(t, h, "i1", "f", 100, 12345678) - h.Close() - - //vv("successful re-open and then Close again of h.") - - // check that we can open a NewHolder on green, on same path, and still see our bits. - // Because the NewHolder is the code that creates and configures TxFactory as blue_green. - cfg := mustHolderConfig() - cfg.StorageConfig.Backend = green - h2 := NewHolder(path, cfg) - PanicOn(h2.Open()) - - testMustHaveBit(t, h2, "i0", "f", rowID, colID) - testMustHaveBit(t, h2, "i1", "f", 100, 200) - testMustHaveBit(t, h2, "i1", "f", 100, 12345678) - h2.Close() - - // verify that blue does not have it. - // open a new holder on path, just looking at blue. - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue - h3 := NewHolder(path, cfg) - PanicOn(h3.Open()) - - testMustNotHaveBit(t, h3, "i0", "f", rowID, colID) - testMustNotHaveBit(t, h3, "i1", "f", 100, 200) - testMustNotHaveBit(t, h3, "i1", "f", 100, 12345678) - - h3.Close() - - // ============================= - // Setup done. On to actual test. - - // Opening in blue_green mode means that once Holder.Open() - // returns without error, the blue and green databases are - // identical. - // Since blue is empty, the blue database will get synched up - // with the green during Holder.Open(). - - // open a holder with path again, now looking at both blue and green. - // The Holder.Open should do the migration from green, populating blue. - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue_green - h4 := NewHolder(path, cfg) - - //vv("about to h4.Open we should populate blue from green") - err = h4.Open() - if expectError { - if err == nil { - panic("expected error since migration to roaring not supported") - } - } else { - PanicOn(err) - } - - testMustHaveBit(t, h4, "i0", "f", rowID, colID) - testMustHaveBit(t, h4, "i1", "f", 100, 200) - testMustHaveBit(t, h4, "i1", "f", 100, 12345678) - - //vv("successfully verified populatingBlueFromGreen with blue_green = '%v'", blue_green) - h4.Close() - os.RemoveAll(path) - } - } -} - -// test the situation where we startup blue_green with existing data and -// go to verify it but blue has more data than green. -// That will also cause query divergence. -func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) { - checked := []string{"roaring", "bolt", "rbf"} - - for _, blue := range checked { - for _, green := range checked { - if blue == green { - continue - } - if blue == "roaring" { - // not supported - continue - } - blue_green := blue + "_" + green - - // ============================= - // Begin setup. - // - // Setup happens with green only. - - h, path, err := makeHolder(t, green) - if err != nil { - t.Fatalf("creating holder: %v", err) - } - defer os.RemoveAll(path) - - //vv("on green, which is '%v'", green) - // we will manually h.Close() below - - // Write bits to separate indexes. - testSetBit(t, h, "i0", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 200) - testSetBit(t, h, "i1", "f", 100, 12345678) - - testOp := testHolderOperator{} - ctx := context.Background() - err = h.Process(ctx, &testOp) - if err != nil { - t.Fatalf("processing holder: %v", err) - } - expected := testHolderOperator{ - indexSeen: 2, indexProcessed: 2, - fieldSeen: 2, fieldProcessed: 2, - viewSeen: 2, viewProcessed: 2, - fragmentSeen: 3, fragmentProcessed: 3, - } - if testOp != expected { - t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp) - } - - // verify data is there - rowID := uint64(100) - colID := uint64(200) - _, _ = rowID, colID - testMustHaveBit(t, h, "i0", "f", rowID, colID) - testMustHaveBit(t, h, "i1", "f", 100, 200) - testMustHaveBit(t, h, "i1", "f", 100, 12345678) - - h.Close() - - // verify that blue does not have it. - // open a new holder on path, just looking at blue. - - //vv("on blue, which is '%v'", blue) - - cfg := mustHolderConfig() - cfg.StorageConfig.Backend = blue - h3 := NewHolder(path, cfg) - PanicOn(h3.Open()) - - testMustNotHaveBit(t, h3, "i0", "f", rowID, colID) - testMustNotHaveBit(t, h3, "i1", "f", 100, 200) - testMustNotHaveBit(t, h3, "i1", "f", 100, 12345678) - - h3.Close() - - // ============================= - // Setup done. On to actual test. - - // Opening in blue_green mode means that once Holder.Open() - // returns without error, the blue and green databases are - // identical. - // Since blue is empty, the blue database will get synched up - // with the green during Holder.Open(). - - //vv("on blue_green, which is '%v'", blue_green) - - // open a holder with path again, now looking at both blue and green. - // The Holder.Open should do the migration from green, populating blue. - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue_green - h4 := NewHolder(path, cfg) - PanicOn(h4.Open()) - - testMustHaveBit(t, h4, "i0", "f", rowID, colID) - testMustHaveBit(t, h4, "i1", "f", 100, 200) - testMustHaveBit(t, h4, "i1", "f", 100, 12345678) - h4.Close() - - // now open just blue, and add a bit to a new index, i2. - //vv("on blue, which is '%v'", blue) - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue - h5 := NewHolder(path, cfg) - PanicOn(h5.Open()) - testSetBit(t, h5, "i2", "f", 500, 777) - - //vv("after adding a bit to blue, we have:") - //h5.DumpAllShards() - - h5.Close() - - // now open blue_green. should get a verification failure - // due to the extra bit in blue. - - // BEGIN verficiation that should ERROR out b/c blue has more data. - - // open a holder with path again, now looking at both blue and green. - // The Holder.Open should verify blue against green and notice the extra bit. - cfg = mustHolderConfig() - cfg.StorageConfig.Backend = blue_green - h6 := NewHolder(path, cfg) - err = h6.Open() - //h6.DumpAllShards() - - if err == nil { - h6.Close() - t.Fatalf("should have had blue-green verification fail on Holder.Open") - } - - h6.Close() - } - } -} - func Test_TxFactory_verifyStringConstantsMatch(t *testing.T) { // txtype.String() method MUST return strings that match - // our const definitions at the top of txfactory.go, or - // else blue-green transactions cannot determine when - // the second transaction is being released in dbshard.go. - check := []txtype{roaringTxn, rbfTxn, boltTxn} - expect := []string{RoaringTxn, RBFTxn, BoltTxn} + // our const definitions at the top of txfactory.go. + check := []txtype{roaringTxn, rbfTxn} + expect := []string{RoaringTxn, RBFTxn} for i, chk := range check { obs := chk.String() if obs != expect[i] { diff --git a/util.go b/util.go index ebbff6e75..6d4b8e09e 100644 --- a/util.go +++ b/util.go @@ -17,19 +17,12 @@ package pilosa // util.go: a place for generic, reusable utilities. import ( - "fmt" - "io/ioutil" "os" - "path/filepath" "reflect" - "sort" - "strings" "syscall" "time" - "unsafe" "github.com/molecula/featurebase/v2/roaring" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck "github.com/pkg/errors" ) @@ -63,229 +56,6 @@ func NilInside(iface interface{}) bool { func highbits(v uint64) uint64 { return v >> 16 } func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) } -func toArray16(a []byte) []uint16 { - return (*[4096]uint16)(unsafe.Pointer(&a[0]))[: len(a)/2 : len(a)/2] -} -func toArray64(a []byte) []uint64 { - return (*[1024]uint64)(unsafe.Pointer(&a[0]))[:1024:1024] -} -func toInterval16(a []byte) []roaring.Interval16 { - return (*[2048]roaring.Interval16)(unsafe.Pointer(&a[0]))[: len(a)/4 : len(a)/4] -} - -func sliceToMap(slc []uint64) (m map[uint64]bool) { - m = make(map[uint64]bool) - for _, v := range slc { - m[v] = true - } - return -} - -// return A - B -func mapDiff(mapA, mapB map[uint64]bool) (r []int) { - for a := range mapA { - _, ok := mapB[a] - if !ok { - r = append(r, int(a)) - } - } - sort.Ints(r) - return -} - -func asInts(a []uint64) (r []int) { - r = make([]int, len(a)) - for i, v := range a { - r[i] = int(v) - } - return -} - -func containerAsString(ckey uint64, rc *roaring.Container) (r string) { - rbm := roaring.NewBitmap() - rbm.Containers.Put(ckey, rc) - return BitmapAsString(rbm) -} - -var _ = containerAsString // happy linter - -func roaringBitmapDiff(a, b *roaring.Bitmap) error { - nA := a.Count() - nB := b.Count() - - slcA := a.Slice() - slcB := b.Slice() - - mapA := sliceToMap(slcA) - mapB := sliceToMap(slcB) - - AminusB := mapDiff(mapA, mapB) - BminusA := mapDiff(mapB, mapA) - - sort.Ints(AminusB) - sort.Ints(BminusA) - - res := fmt.Sprintf("nA = %v; nB = %v;\n", nA, nB) - ndiff := 0 - if nA != nB { - ndiff++ - } - - if len(AminusB) > 0 { - res += fmt.Sprintf("==> AminusB = (len %v) '%#v'; ", len(AminusB), AminusB) - ndiff++ - } - if len(BminusA) > 0 { - res += fmt.Sprintf("\n==> BminusA = (len %v) '%#v'; ", len(BminusA), BminusA) - ndiff++ - } - if ndiff == 0 { - return nil - } - res += fmt.Sprintf("\n ==> A = '%#v'\n ==> B = '%#v'", asInts(slcA), asInts(slcB)) - return errors.New(res) -} - -func dirAsString(path string) (r string) { - r = fmt.Sprintf("dump of directory '%v':\n", path) - files, err := ioutil.ReadDir(path) - PanicOn(err) - for _, f := range files { - r += f.Name() + "\n" - } - return r -} - -var _ = dirAsString // happy linter - -var _ = zeroKeyContainerAsString // happy linter - -// for debugging -func zeroKeyContainerAsString(ct *roaring.Container) (r string) { - cts := roaring.NewSliceContainers() - cts.Put(0, ct) - rbm := &roaring.Bitmap{Containers: cts} - r = fmt.Sprintf("[%v]:", containerTypeNames[roaring.ContainerType(ct)]) + BitmapAsString(rbm) - return -} - -var containerTypeNames = map[byte]string{ - roaring.ContainerArray: "array", - roaring.ContainerBitmap: "bitmap", - roaring.ContainerRun: "run", -} - -func BitmapAsString(rbm *roaring.Bitmap) (r string) { - r = "c(" - slc := rbm.Slice() - width := 0 - s := "" - for _, v := range slc { - if width == 0 { - s = fmt.Sprintf("%v", v) - } else { - s = fmt.Sprintf(", %v", v) - } - width += len(s) - r += s - if width > 70 { - r += ",\n" - width = 0 - } - } - if width == 0 && len(r) > 2 { - r = r[:len(r)-2] - } - return r + ")" -} - -// fromArray16 converts to an 8KB page -func fromArray16(a []uint16) []byte { - if len(a) == 0 { - return []byte{} - } - if len(a) > 4096 { - PanicOn(fmt.Sprintf("cannot put more than 4096 integers into an array container: %v too big", len(a))) - } - return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2] -} - -// fromArray64 converts to an 8KB page -func fromArray64(a []uint64) []byte { - if len(a) == 0 { - return []byte{} - } - return (*[8192]byte)(unsafe.Pointer(&a[0]))[:8192:8192] -} - -// fromInterval16 converts to 8KB page -func fromInterval16(a []roaring.Interval16) []byte { - if len(a) == 0 { - return []byte{} - } - if len(a) > 2048 { - PanicOn(fmt.Sprintf("cannot put more than 2048 roaring.Interval16 into a container: %v too big", len(a))) - } - return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4] -} - -// DiskUse reports the total bytes uses by all files under root -// that match requiredSuffix. requiredSuffix can be empty string. -// Space used by directories is not counted. -func DiskUse(root string, requiredSuffix string) (tot int, err error) { - if !DirExists(root) { - return -1, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root) - } - - err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if info == nil { - PanicOn(fmt.Sprintf("info was nil for path = '%v'", path)) - } - if info.IsDir() { - // skip the size of directories themselves, only summing files. - } else { - sz := info.Size() - if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) { - tot += int(sz) - } - } - return nil - }) - return -} - -// rootDir must exist. Return the size in bytes of the largest sub-directory -// that has the required suffix. The largestSize is from DiskUse() called -// on the sub-dir. DiskUse only counts file size, nothing for directory inodes. -func SubdirLargestDirWithSuffix(rootDir, requiredDirSuffix string) (exists bool, largestSize int, err error) { - if !DirExists(rootDir) { - return false, -1, fmt.Errorf("SubdirExistsWithSuffix error: root directory '%v' not found", rootDir) - } - - err = filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { - if info == nil { - PanicOn(fmt.Sprintf("info was nil for path = '%v'", path)) - } - - if info.IsDir() && strings.HasSuffix(path, requiredDirSuffix) { - exists = true - size, err := DiskUse(path, "") - if err != nil { - // disk error? report it - return err - } - if size > largestSize { - largestSize = size - } - } - return nil - }) - if err != nil { - return exists, -1, err - } - return -} - // called by Holder.hasRoaringData() func roaringFragmentHasData(path string, index, field, view string, shard uint64) (hasData bool, err error) { diff --git a/utils_internal_test.go b/utils_internal_test.go index 76c3aa66f..fa9b666a8 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -15,47 +15,17 @@ package pilosa import ( - "bytes" "fmt" "testing" "time" pnet "github.com/molecula/featurebase/v2/net" - "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/testhook" "github.com/molecula/featurebase/v2/topology" - . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck ) // utilities used by tests -// mustAddR is a helper for calling roaring.Container.Add() in tests to -// keep the linter happy that we are checking the error. -func mustAddR(changed bool, err error) { - PanicOn(err) -} - -// mustRemove is a helper for calling Tx.Remove() in tests to -// keep the linter happy that we are checking the error. -func mustRemove(changeCount int, err error) { - PanicOn(err) -} - -func getTestBitmapAsRawRoaring(bitsToSet ...uint64) []byte { - b := roaring.NewBitmap() - changed := b.DirectAddN(bitsToSet...) - n := len(bitsToSet) - if changed != n { - panic(fmt.Sprintf("changed=%v but bitsToSet len = %v", changed, n)) - } - buf := bytes.NewBuffer(make([]byte, 0, 100000)) - _, err := b.WriteTo(buf) - if err != nil { - panic(err) - } - return buf.Bytes() -} - // NewTestCluster returns a cluster with n nodes and uses a mod-based hasher. func NewTestCluster(tb testing.TB, n int) *cluster { path, err := testhook.TempDir(tb, "pilosa-cluster-") diff --git a/view.go b/view.go index ac94f64d8..911a2d19c 100644 --- a/view.go +++ b/view.go @@ -158,26 +158,28 @@ func (v *view) openWithShardSet(ss *shardSet) error { if nGoro < 4 { nGoro = 4 } - pj := newParallelJobs(nGoro) + var eg errgroup.Group + throttle := make(chan struct{}, nGoro) + for i := range frags { // create a new variable frag on each time through // the loop (instead of i, frag := range frags) // so that the closure run on the // goroutine has its own variable. frag := frags[i] - accepted := pj.run(func(worker int) error { + throttle <- struct{}{} + eg.Go(func() error { + defer func() { + <-throttle + }() if err := frag.Open(); err != nil { return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err) } return nil }) - if !accepted { - // have error/shutting down the pj, so stop - break - } } - err := pj.waitForFinish() + err := eg.Wait() if err != nil { return err }