scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1731 from molecula/core930

Browse source 
[FB-930] remove bolt backend, bluegreentx, and a ton of unused API surface
This commit is contained in:
seebs 2021-10-26 13:37:16 -05:00 • committed by GitHub
commit 6cb9ce0315
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
45 changed files with 236 additions and 7482 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -9,6 +9,7 @@ release-pilosa-fsck.*.*.tar.gz
/log.*
/tourna.log.*
pilosa
/featurebase
*.dot
.idea/
.*.swp

12
api.go
View file

@ -1510,9 +1510,7 @@ func addClearToImportOptions(opts []ImportOption) []ImportOption {
return append(opts, OptImportOptionsClear(true))
}
// Import avoids re-writing a bajillion tests to be transaction-aware by allowing a nil pQcx.
// It is convenient for some tests, particularly those in loops, to pass a nil qcx and
// treat the Import as having been commited when we return without error. We make it so.
// Import does the top-level importing.
func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts ...ImportOption) (err error) {
if req.Clear {
opts = addClearToImportOptions(opts)
@ -1648,8 +1646,8 @@ func (api *API) ImportWithTx(ctx context.Context, qcx *Qcx, req *ImportRequest,
return errors.Wrap(err, "committing")
}
// ImportValue avoids re-writing a bajillion tests by allowing a nil pQcx.
// Then we will commit before returning.
// ImportValue is a wrapper around the common code in ImportValueWithTx, which
// currently just translates req.Clear into a clear ImportOption.
func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueRequest, opts ...ImportOption) error {
if req.Clear {
opts = addClearToImportOptions(opts)
@ -2654,9 +2652,7 @@ func (api *API) RestoreShard(ctx context.Context, indexName string, shard uint64
if err != nil {
return err
}
//need to find the path to the db
//will not work on blue green
db := dbs.W[0]
db := dbs.W
finalPath := db.Path() + "/data"
tempPath := finalPath + ".tmp"
o, err := os.OpenFile(tempPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)

219
barrier.go
View file

@ -1,219 +0,0 @@
// home https://github.com/glycerine/lmdb-go
// Copyright (c) 2020, the lmdb-go authors
// Copyright (c) 2015, Bryan Matsuo
// All rights reserved.
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
// Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
// Neither the name of the author nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package pilosa
import (
"github.com/glycerine/idem"
)
// Barrier allows us to temporarily halt all readers, so that
// a writer can commit alone and thus compact the db.
// The Barrier starts unblocked, alllowing passage to any
// caller of WaitAtGate().
type Barrier struct {
wait chan *appointment // send upon entering the waiting room.
halt *idem.Halter
blockReqCh chan *blockReq
unblockCh chan *unblock
}
type blockReq struct {
count int
done chan struct{}
}
func newBlockReq(count int) *blockReq {
return &blockReq{
count: count,
done: make(chan struct{}),
}
}
type appointment struct {
id int
done chan struct{}
}
func newAppointment(id int) *appointment {
return &appointment{
id: id,
done: make(chan struct{}),
}
}
// NewBarrier is either open, allowing immediate passage,
// or blocked, halting all callers at WaitAtGate()
// until the barrier is opened. By default it is open.
//
// Barrier.Close() must be called when the barrier
// is no longer needed to avoid a goroutine leak.
func NewBarrier() (b *Barrier) {
b = &Barrier{
wait: make(chan *appointment), // waiters indicate they are waiting for the gate by sending here.
halt: idem.NewHalter(),
blockReqCh: make(chan *blockReq),
unblockCh: make(chan *unblock),
}
go func() {
defer b.halt.Done.Close()
var waitlist []*appointment
var curBlockReq *blockReq
for {
select {
case br := <-b.blockReqCh:
if br.count == 0 {
close(br.done)
continue
}
if curBlockReq == nil {
// good, changing state from open to closed barrier.
} else {
panic("got 2nd block request atop of first")
}
curBlockReq = br
//vv("barrier: request to block for %v waiters", br.count)
if len(waitlist) != 0 {
panic("had waiters when we were open, internal/client bug")
}
case appt := <-b.wait:
//vv("barrier.wait sees appt = '%#v' and curBlockReq = '%#v'", appt, curBlockReq)
if curBlockReq == nil {
close(appt.done)
continue
}
waitlist = append(waitlist, appt)
n := len(waitlist)
th := curBlockReq.count
if th < 0 {
// infinite waiters. we block everybody until we
// see an unblock request.
continue
}
if n >= th {
close(curBlockReq.done)
curBlockReq = nil
}
case ub := <-b.unblockCh:
for _, appt := range waitlist {
close(appt.done)
}
waitlist = nil
curBlockReq = nil
close(ub.done)
case <-b.halt.ReqStop.Chan:
return
}
}
}()
return
}
// WaitAtGate will return immediately
// if the barrier is unblocked. Otherwise
// it will not return until another
// goroutine unblocks the barrier.
func (b *Barrier) WaitAtGate(id int) {
appt := newAppointment(id)
select {
case b.wait <- appt:
select {
case <-appt.done:
case <-b.halt.ReqStop.Chan:
}
case <-b.halt.ReqStop.Chan:
}
}
// Close should be called to stop the
// barrier's background goroutine when
// you are done using the barrier.
func (b *Barrier) Close() {
b.halt.ReqStop.Close()
<-b.halt.Done.Chan
}
type unblock struct {
done chan struct{}
}
func newUnblock() *unblock {
return &unblock{
done: make(chan struct{}),
}
}
// Unblock lets all waiting goroutines resume execution.
func (b *Barrier) UnblockReaders() {
ub := newUnblock()
select {
case b.unblockCh <- ub:
select {
case <-ub.done:
case <-b.halt.ReqStop.Chan:
}
case <-b.halt.ReqStop.Chan:
}
}
// BlockUntil is called with a count, the
// number of waiters required to be present and waiting
// at the gate before call returns.
// A count of < 0 will return immediately and raise
// the barrier to any number of arriving readers.
// A count of 0 is a no-op.
//
// Otherwise we raise the barrier
// and wait until we have seen count other goroutines waiting
// on it.
//
// We return without releasing the waiters. Call
// Open when you want them to resume.
func (b *Barrier) BlockUntil(count int) {
if count == 0 {
return
}
req := newBlockReq(count)
b.blockReqCh <- req
if count > 0 {
<-req.done
}
}
// BlockAllReadersNoWait raises the barrier to
// an infinite number of waiters and returns immediately
// to the caller.
func (b *Barrier) BlockAllReadersNoWait() {
req := newBlockReq(-1) // -1 means block any number of readers.
b.blockReqCh <- req
// don't wait. <-req.done
}

90
barrier_test.go
View file

@ -1,90 +0,0 @@
// home https://github.com/glycerine/lmdb-go
// Copyright (c) 2020, the lmdb-go authors
// Copyright (c) 2015, Bryan Matsuo
// All rights reserved.
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
// Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
// Neither the name of the author nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package pilosa
import (
"fmt"
"sync/atomic"
"testing"
"time"
)
func TestBarrierHolds(t *testing.T) {
b := NewBarrier()
defer b.Close()
released := int64(0)
waiter := func(i int) {
b.WaitAtGate(i)
//vv("goro %v is released", i)
atomic.AddInt64(&released, 1)
}
for i := 0; i < 3; i++ {
go waiter(i)
}
time.Sleep(time.Second)
r := atomic.SwapInt64(&released, 0)
if r != 3 {
panic("open barrier held back goro")
}
//vv("good: barrier started open")
//seenAll := make(chan bool)
b.BlockAllReadersNoWait()
for i := 0; i < 3; i++ {
go waiter(i)
}
time.Sleep(time.Second)
r = atomic.SwapInt64(&released, 0)
if r != 0 {
panic("bad: barrier did not hold back goro")
}
//vv("good: barrier of 4 did not release on 3")
go waiter(4)
time.Sleep(time.Second)
r = atomic.SwapInt64(&released, 0)
if r != 0 {
panic(fmt.Sprintf("bad: barrier did not hold back goro, should wait for unblock. r = %v", r))
}
b.UnblockReaders()
time.Sleep(time.Second)
r = atomic.SwapInt64(&released, 0)
if r != 4 {
panic("bad: unblock should have released 4 goro")
}
}

1028
bluegreentx.go
View file

File diff suppressed because it is too large Load diff

100
bluegreentx_test.go
View file

@ -1,100 +0,0 @@
// Copyright 2020 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"bytes"
"context"
"io"
"io/ioutil"
"os"
"strings"
"testing"
cryrand "crypto/rand"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
)
var _ = context.Background
var _ = os.Open
var _ = strings.Split
func TestMultiReaderB(t *testing.T) {
// MultiReaderB should read identical chunks of bytes from both its "a" and "b"
// member io.Readers, else it should panic. This should hold for
// varying sizes of inputs.
for n := 1 << 5; n < (1 << 18); n = n*2 - 13 {
src := io.LimitReader(cryrand.Reader, int64(n))
a := make([]byte, n)
nr := 0
for nr < n {
na, err := src.Read(a)
PanicOn(err)
nr += na
}
if nr != n {
panic("short read")
}
b := make([]byte, n)
copy(b, a)
if !bytes.Equal(a, b) {
panic("test prep failed")
}
m := &MultiReaderB{
a: ioutil.NopCloser(bytes.NewBuffer(a)),
b: ioutil.NopCloser(bytes.NewBuffer(b)),
}
// should not trigger the internal panic of MultiReadB
ncp, err := io.Copy(ioutil.Discard, m)
PanicOn(err)
if ncp != int64(n) {
panic("short copy")
}
for victim := 0; victim < n; victim += 7 {
copy(b, a)
if victim%2 == 0 {
// corrupt b
b[victim] = (b[victim] + 1) % 255
} else {
// corrupt a
a[victim] = (a[victim] + 1) % 255
}
m = &MultiReaderB{
a: ioutil.NopCloser(bytes.NewBuffer(a)),
b: ioutil.NopCloser(bytes.NewBuffer(b)),
}
helperShouldPanicOnCopy(m)
}
}
}
func helperShouldPanicOnCopy(m *MultiReaderB) {
// differences in bytes read should be noticed
defer func() {
r := recover()
if r == nil {
panic("expected panic on byte difference but didn't see it")
}
}()
_, _ = io.Copy(ioutil.Discard, m)
}

1612
bolt.go
View file

File diff suppressed because it is too large Load diff

1270
bolt_test.go
View file

File diff suppressed because it is too large Load diff

71
catcher.go
View file

@ -15,9 +15,6 @@
package pilosa
import (
"fmt"
"io"
"github.com/molecula/featurebase/v2/roaring"
txkey "github.com/molecula/featurebase/v2/short_txkey"
. "github.com/molecula/featurebase/v2/vprint"
@ -42,40 +39,18 @@ func init() {
var _ Tx = (*catcherTx)(nil)
func (c *catcherTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
c.b.IncrementOpN(index, field, view, shard, changedN)
}
func (c *catcherTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
return c.b.NewTxIterator(index, field, view, shard)
}
func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
func (c *catcherTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see ImportRoaringBits() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
}
func (c *catcherTx) Dump(short bool, shard uint64) {
c.b.Dump(short, shard)
}
func (c *catcherTx) Readonly() bool {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Readonly() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Readonly()
}
func (tx *catcherTx) Pointer() string {
return fmt.Sprintf("%p", tx)
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
}
func (c *catcherTx) Rollback() {
@ -143,14 +118,6 @@ func (c *catcherTx) RemoveContainer(index, field, view string, shard uint64, key
return c.b.RemoveContainer(index, field, view, shard, key)
}
func (c *catcherTx) UseRowCache() bool {
return c.b.UseRowCache()
}
func (c *catcherTx) IsDone() bool {
return c.b.IsDone()
}
func (c *catcherTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
defer func() {
@ -250,17 +217,6 @@ func (c *catcherTx) Min(index, field, view string, shard uint64) (uint64, bool,
return c.b.Min(index, field, view, shard)
}
func (c *catcherTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.UnionInPlace(index, field, view, shard, others...)
}
func (c *catcherTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
defer func() {
@ -283,33 +239,10 @@ func (c *catcherTx) OffsetRange(index, field, view string, shard, offset, start,
return c.b.OffsetRange(index, field, view, shard, offset, start, end)
}
func (c *catcherTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
}
func (c *catcherTx) Type() string {
return c.b.Type()
}
func (c *catcherTx) Group() *TxGroup {
return c.b.Group()
}
func (c *catcherTx) Options() Txo {
return c.b.Options()
}
// Sn retreives the serial number of the Tx.
func (c *catcherTx) Sn() int64 {
return c.b.Sn()
}
func (c *catcherTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
}

50
cluster_internal_test.go
View file

@ -17,7 +17,6 @@ package pilosa
import (
"fmt"
"math/rand"
"net"
"reflect"
"strings"
"testing"
@ -30,57 +29,8 @@ import (
"github.com/molecula/featurebase/v2/testhook"
"github.com/molecula/featurebase/v2/topology"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
)
// GlobalPortMap avoids many races and port conflicts when setting
// up ports for test clusters. Used for tests only.
var globalPortMap *GlobalPortMapper
func init() {
globalPortMap = NewGlobalPortMapper(300)
}
// GlobalPortMapper maintains a pool of available ports by
// holding them open until GetPort() is called.
type GlobalPortMapper struct {
availPorts map[int]net.Listener
}
// reserve n ports
func NewGlobalPortMapper(n int) (pm *GlobalPortMapper) {
pm = &GlobalPortMapper{
availPorts: make(map[int]net.Listener),
}
for i := 0; i < n; i++ {
lsn, err := net.Listen("tcp", ":0")
if err != nil {
panic(errors.Wrap(err, "trying to listen on ephemeral port"))
}
r := lsn.Addr()
port := r.(*net.TCPAddr).Port
pm.availPorts[port] = lsn
}
return
}
func (pm *GlobalPortMapper) GetPort() (port int, err error) {
for port, lsn := range pm.availPorts {
lsn.Close()
return port, nil
}
return -1, fmt.Errorf("no more ports available")
}
func (pm *GlobalPortMapper) MustGetPort() int {
port, err := pm.GetPort()
if err != nil {
panic(err)
}
return port
}
// Ensure that fragCombos creates the correct fragment mapping.
func TestFragCombos(t *testing.T) {
uri0, err := pnet.NewURIFromAddress("host0")

2
cmd/roaring-migrate/main.go
View file

@ -310,7 +310,7 @@ func Migrate(dataDir, backupPath string) error {
return err
}
key := string(txkey.Prefix(index, field, view, shard))
_, _, err = tx.ImportRoaringBits(key, itr, clear, log, rowSize, nil)
_, _, err = tx.ImportRoaringBits(key, itr, clear, log, rowSize)
if err != nil {
tx.Rollback()
return err

2
ctl/server.go
View file

@ -94,7 +94,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
// over-ride.
// TODO: the comment above was carried over from the PILOSA_TXSRC flag, but
// we should confirm that this still applies.
flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring, rbf, bolt, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend))
flags.StringVar(&srv.Config.Storage.Backend, "storage.backend", storage.DefaultBackend, fmt.Sprintf("transaction/storage to use: one of roaring or rbf. The default is: %v. The env var PILOSA_STORAGE_BACKEND is over-ridden by --storage.backend option on the command line.", storage.DefaultBackend))
flags.BoolVar(&srv.Config.Storage.FsyncEnabled, "storage.fsync", true, "enable fsync fully safe flush-to-disk")
// RowcacheOn

638
dbshard.go
View file

@ -57,12 +57,10 @@ type DBIndex struct {
type DBWrapper interface {
NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error)
DeleteDBPath(dbs *DBShard) error
Close() error
DeleteFragment(index, field, view string, shard uint64, frag interface{}) error
DeleteField(index, field, fieldPath string) error
OpenListString() string
OpenSnList() (sns []int64)
Path() string
HasData() (has bool, err error)
SetHolder(h *Holder)
@ -82,134 +80,52 @@ type DBShard struct {
Shard uint64
Open bool
// With RWMutex, the blue-green Tx can start and commit
// atomically.
mut sync.RWMutex
types []txtype
stypes []string
typ txtype
styp string
hasRoaring bool // if either of the types is roaringTxn
W []DBWrapper
W DBWrapper
ParentDBIndex *DBIndex
idx *Index
per *DBPerShard
useOpenList int
closed bool
isBlueGreen bool
closed bool
}
func (dbs *DBShard) DeleteFragment(index, field, view string, shard uint64, frag interface{}) (err error) {
for _, w := range dbs.W {
err = w.DeleteFragment(index, field, view, shard, frag)
if err != nil {
return err
}
if index != dbs.Index {
return fmt.Errorf("DeleteFragment called on DBShard for %q with index %q", dbs.Index, index)
}
return
if shard != dbs.Shard {
return fmt.Errorf("DeleteFragment called on DBShard for %d with shard %d", dbs.Shard, shard)
}
return dbs.W.DeleteFragment(index, field, view, shard, frag)
}
func (dbs *DBShard) DeleteFieldFromStore(index, field, fieldPath string) (err error) {
for _, w := range dbs.W {
err = w.DeleteField(index, field, fieldPath)
if err != nil {
return err
}
if index != dbs.Index {
return fmt.Errorf("DeleteFieldFromStore called on DBShard for %q with index %q", dbs.Index, index)
}
return
return dbs.W.DeleteField(index, field, fieldPath)
}
func (dbs *DBShard) Close() (err error) {
for _, w := range dbs.W {
err = w.Close()
if err != nil {
return err
}
}
dbs.closed = true
return
}
// Cleanup must be called at every commit/rollback of a Tx, in
// order to release the read-write mutex that guarantees a single
// writer at a time. Each tx must take care to call cleanup()
// exactly once. examples:
// tx.o.dbs.Cleanup(tx)
// tx.Options().dbs.Cleanup(tx)
//
func (dbs *DBShard) Cleanup(tx Tx) {
if dbs == nil {
return // some tests are using Tx only, no dbs available.
}
//vv("gid %v top of DBShard %v Cleanup for tx.Sn = %v; dbs=%p; is 2nd: %v; type='%v'; dbs.stypes='%#v'", curGID(), dbs.Shard, tx.Sn(), dbs, tx.Type() == dbs.stypes[1], tx.Type(), dbs.stypes)
if !dbs.hasRoaring {
if dbs.isBlueGreen {
// only release on the 2nd Tx's cleanup
if tx.Type() == dbs.stypes[1] {
if tx.Readonly() {
dbs.mut.RUnlock()
//vv("gid %v released read-lock on shard %v", curGID(), dbs.Shard)
} else {
dbs.mut.Unlock()
//vv("gid %v released write-lock on shard %v", curGID(), dbs.Shard)
}
}
}
}
return dbs.W.Close()
}
func (dbs *DBShard) NewTx(write bool, initialIndexName string, o Txo) (tx Tx, err error) {
if dbs.isBlueGreen {
// enforce only one writer at a time. The dbs.mut is held until
// the Tx finishes. This makes the two Tx in the blue-green Tx atomic.
if !dbs.hasRoaring {
if write {
//vv("shard %v about to write lock by gid %v; stack =\n%v", dbs.Shard, curGID(), stack())
dbs.mut.Lock()
//vv("shard %v was write locked by gid %v; stack =\n%v", dbs.Shard, curGID(), stack())
} else {
//vv("shard %v about to be read locked by gid %v; stack=\n%v", dbs.Shard, curGID(), stack())
dbs.mut.RLock()
//vv("shard %v was read locked by gid %v; stack=\n%v", dbs.Shard, curGID(), stack())
}
}
if initialIndexName != dbs.Index {
return nil, fmt.Errorf("NewTx called on DBShard for %q with index %q", dbs.Index, initialIndexName)
}
if o.dbs != dbs {
PanicOn(fmt.Sprintf("TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs))
return nil, fmt.Errorf("dbs mismatch: TxFactory.NewTx() should have set o.dbs(%p) to equal dbs(%p)", o.dbs, dbs)
}
if o.Shard != dbs.Shard {
PanicOn(fmt.Sprintf("shard disagreement! o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard)))
return nil, fmt.Errorf("shard disagreement: o.Shard='%v' but dbs.Shard='%v'", int(o.Shard), int(dbs.Shard))
}
var txns []Tx
for _, w := range dbs.W {
tx, err = w.NewTx(write, initialIndexName, o)
if err != nil {
return nil, err
}
txns = append(txns, tx)
}
if len(txns) == 1 {
return
}
// blue green
tx, err = dbs.per.txf.newBlueGreenTx(txns[0], txns[1], o.Index, o), nil
//vv("dbshard returning blue-green tx sn %v", tx.Sn())
return
}
func (dbs *DBShard) DeleteDBPath() (err error) {
for _, w := range dbs.W {
err = w.DeleteDBPath(dbs)
if err != nil {
return err
}
}
return
return dbs.W.NewTx(write, initialIndexName, o)
}
type flatkey struct {
@ -228,34 +144,26 @@ type DBPerShard struct {
// Easily see how many we have.
Flatmap map[flatkey]*DBShard
types []txtype
typ txtype
hasRoaring bool
txf *TxFactory
holder *Holder
// which of our types is not-roaring, since
// roaring doesn't keep a list of open Tx sn.
// or default to the 2nd.
useOpenList int
// cache the shards per index to avoid excessive
// directory scans of the index directory. Keep per
// txtype to allow blue-green migrate open to be fast too.
// directory scans of the index directory.
// Keep it up-to-date as we add shards to avoid doing
// a filesystem rescan on new shard creation.
//
// txtype -> index -> *shardSet
index2shards map[txtype]map[string]*shardSet
isBlueGreen bool
// index -> *shardSet
index2shards map[string]*shardSet
StorageConfig *storage.Config
RBFConfig *rbfcfg.Config
}
func newIndex2Shards() (r map[txtype]map[string]*shardSet) {
r = make(map[txtype]map[string]*shardSet)
func newIndex2Shards() (r map[string]*shardSet) {
r = make(map[string]*shardSet)
return
}
@ -350,51 +258,6 @@ func newShardSetFromMap(m map[uint64]bool) *shardSet {
}
}
// HasData returns true if the database has at least one key.
// For roaring it returns true if we a fragment stored.
// The `which` argument is the index into the per.W slice. 0 for blue, 1 for green.
// If you pass 1, be sure you have a blue-green configuration.
func (per *DBPerShard) HasData(which int) (hasData bool, err error) {
// has to aggregate across all available DBShard for each index and shard.
if per.types[which] == roaringTxn {
return per.RoaringHasData() // this needs to be made accurate
}
for _, v := range per.Flatmap {
hasData, err = v.W[which].HasData()
if err != nil {
return
}
if hasData {
return
}
}
return
}
func (per *DBPerShard) RoaringHasData() (bool, error) {
idxs := per.holder.Indexes()
const requireData = true
for _, idx := range idxs {
shards, err := per.TypedDBPerShardGetShardsForIndex(roaringTxn, idx, "", requireData)
if err != nil {
return false, err
}
if len(shards) > 0 {
return true, nil
}
}
return false, nil
}
func (per *DBPerShard) ListOpenString() (r string) {
for _, v := range per.Flatmap {
r += v.HolderPath + " -> " + v.W[per.useOpenList].OpenListString() + "\n"
}
return
}
func (per *DBPerShard) LoadExistingDBs() (err error) {
idxs := per.holder.Indexes()
@ -414,37 +277,24 @@ func (per *DBPerShard) LoadExistingDBs() (err error) {
return
}
func (txf *TxFactory) NewDBPerShard(types []txtype, holderDir string, holder *Holder) (d *DBPerShard) {
func (txf *TxFactory) NewDBPerShard(typ txtype, holderDir string, holder *Holder) (d *DBPerShard) {
if holder.cfg == nil || holder.cfg.RBFConfig == nil || holder.cfg.StorageConfig == nil {
PanicOn("must have holder.cfg.RBFConfig and holder.cfg.StorageConfig set here")
}
useOpenList := 0
hasRoaring := false
if types[0] == roaringTxn {
if typ == roaringTxn {
hasRoaring = true
}
if len(types) == 2 {
// blue-green, avoid the empty roaring Tx open list.
// Prefer B's open list if neither is roaring.
if types[0] == roaringTxn || types[1] != roaringTxn {
useOpenList = 1
}
if types[1] == roaringTxn {
hasRoaring = true
}
}
d = &DBPerShard{
types: types,
typ: typ,
HolderDir: holderDir,
holder: holder,
dbh: NewDBHolder(),
Flatmap: make(map[flatkey]*DBShard),
txf: txf,
useOpenList: useOpenList,
hasRoaring: hasRoaring,
isBlueGreen: len(types) > 1,
index2shards: newIndex2Shards(),
StorageConfig: holder.cfg.StorageConfig,
RBFConfig: holder.cfg.RBFConfig,
@ -469,14 +319,12 @@ func (per *DBPerShard) DeleteIndex(index string) (err error) {
if err != nil {
return errors.Wrap(err, "DBPerShard.DeleteIndex dbs.Close()")
}
for _, ty := range per.types {
path := dbs.pathForType(ty)
err = os.RemoveAll(path)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("DBPerShard.DeleteIndex os.RemoveAll('%v')", path))
}
delete(per.index2shards[ty], index)
path := dbs.pathForType(per.typ)
err = os.RemoveAll(path)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("DBPerShard.DeleteIndex os.RemoveAll('%v')", path))
}
delete(per.index2shards, index)
}
// allow the index to be created again anew.
@ -502,10 +350,8 @@ func (per *DBPerShard) DeleteFieldFromStore(index, field, fieldPath string) (err
return nil
}
for _, dbs := range dbi.Shard {
for _, w := range dbs.W {
if e := w.DeleteField(index, field, fieldPath); e != nil && err == nil {
err = errors.Wrap(e, "DeleteFieldFromStore()")
}
if e := dbs.W.DeleteField(index, field, fieldPath); e != nil && err == nil {
err = errors.Wrap(e, "DeleteFieldFromStore()")
}
}
return err
@ -521,46 +367,6 @@ func (per *DBPerShard) DeleteFragment(index, field, view string, shard uint64, f
return dbs.DeleteFragment(index, field, view, shard, frag)
}
func (dbs *DBShard) DumpAll() {
short := false
fmt.Printf("\n============= begin DumpAll dbs=%p index='%v', shard=%v ========\n", dbs, dbs.Index, int(dbs.Shard))
for i, ty := range dbs.types {
_ = i
tx, err := dbs.W[i].NewTx(!writable, "", Txo{Index: dbs.idx})
PanicOn(err)
defer tx.Rollback()
fmt.Printf("\n============= dumping dbs.W[%v] %v ========\n", i, ty)
tx.Dump(short, dbs.Shard)
switch ty {
case roaringTxn:
case rbfTxn:
case boltTxn:
default:
PanicOn(fmt.Sprintf("unknown txtyp: '%v'", ty))
}
}
fmt.Printf("\n============= end of DumpAll index='%v', shard=%v ========\n", dbs.Index, int(dbs.Shard))
}
func (per *DBPerShard) DumpAll() {
per.Mu.Lock()
defer per.Mu.Unlock()
found1 := false
for _, dbi := range per.dbh.Index {
for _, dbs := range dbi.Shard {
if dbs.Open {
found1 = true
dbs.DumpAll()
}
}
}
if !found1 {
AlwaysPrintf("DBPerShard.DumpAll() sees no databases. dir='%v'", per.HolderDir)
}
}
// if you know the shard, you can use this
// pathForType and prefixForType must be kept in sync!
func (dbs *DBShard) pathForType(ty txtype) string {
@ -570,11 +376,6 @@ func (dbs *DBShard) pathForType(ty txtype) string {
// is a no-op anyhow. so doesn't need to be correct atm.
path := dbs.HolderPath + sep + dbs.Index + sep + backendsDir + sep + ty.DirectoryName() + sep + fmt.Sprintf("shard.%04v", dbs.Shard)
if ty == boltTxn {
// special case:
// bolt doesn't use a directory like the others, just a direct path.
path += sep + "bolt.db"
}
return path
}
@ -593,23 +394,13 @@ var ErrNoData = fmt.Errorf("no data")
//
// Caller must hold per.Mu.Lock() already.
func (per *DBPerShard) updateIndex2ShardCacheWithNewShard(dbs *DBShard) {
for _, ty := range dbs.types {
mapIndex2shardSet, ok := per.index2shards[ty]
if !ok {
mapIndex2shardSet = make(map[string]*shardSet)
per.index2shards[ty] = mapIndex2shardSet
}
// INVAR: mapIndex2shardSet is good, but may be an empty map
shardset, ok := mapIndex2shardSet[dbs.Index]
if !ok {
shardset = newShardSet()
mapIndex2shardSet[dbs.Index] = shardset
}
// INVAR: shardset is present, not nil; a map that can be added to.
shardset.add(dbs.Shard)
shardset, ok := per.index2shards[dbs.Index]
if !ok {
shardset = newShardSet()
per.index2shards[dbs.Index] = shardset
}
// INVAR: shardset is present, not nil; a map that can be added to.
shardset.add(dbs.Shard)
}
func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs *DBShard, err error) {
@ -629,76 +420,47 @@ func (per *DBPerShard) unprotectedGetDBShard(index string, shard uint64, idx *In
}
dbs, ok = dbi.Shard[shard]
if dbs != nil && dbs.closed {
if len(per.types) == 1 && per.types[0] == roaringTxn {
// roaring txn are nil/fake anyway. Don't freak out.
} else {
PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.types[0]='%v'; len(per.types)=%v", dbs, per.types[0], len(per.types)))
// roaring txn are nil/fake anyway. Don't freak out.
if per.typ != roaringTxn {
PanicOn(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.typ='%v'", dbs, per.typ))
}
}
if !ok {
dbs = &DBShard{
types: per.types,
typ: per.typ,
ParentDBIndex: dbi,
Index: index,
Shard: shard,
HolderPath: per.HolderDir,
idx: idx,
per: per,
useOpenList: per.useOpenList,
hasRoaring: per.hasRoaring,
isBlueGreen: len(per.types) > 1,
}
dbs.stypes = make([]string, len(per.types))
for i, ty := range per.types {
dbs.stypes[i] = ty.String()
}
dbs.styp = per.typ.String()
dbi.Shard[shard] = dbs
per.updateIndex2ShardCacheWithNewShard(dbs)
}
if !dbs.Open {
var registry DBRegistry
for _, ty := range dbs.types {
switch ty {
case roaringTxn:
registry = globalRoaringReg
case rbfTxn:
registry = globalRbfDBReg
registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig)
case boltTxn:
registry = globalBoltReg
default:
PanicOn(fmt.Sprintf("unknown txtyp: '%v'", ty))
}
path := dbs.pathForType(ty)
w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.StorageConfig)
PanicOn(err)
h := idx.Holder()
w.SetHolder(h)
dbs.Open = true
if w != nil && len(dbs.W) == 0 {
per.Flatmap[flatkey{index: index, shard: shard}] = dbs
}
dbs.W = append(dbs.W, w)
switch dbs.typ {
case roaringTxn:
registry = globalRoaringReg
case rbfTxn:
registry = globalRbfDBReg
registry.(*rbfDBRegistrar).SetRBFConfig(per.RBFConfig)
default:
PanicOn(fmt.Sprintf("unknown txtyp: '%v'", dbs.typ))
}
path := dbs.pathForType(dbs.typ)
w, err := registry.OpenDBWrapper(path, DetectMemAccessPastTx, per.StorageConfig)
PanicOn(err)
h := idx.Holder()
w.SetHolder(h)
dbs.Open = true
per.Flatmap[flatkey{index: index, shard: shard}] = dbs
dbs.W = w
}
return
}
func (per *DBPerShard) Del(dbs *DBShard) (err error) {
per.Mu.Lock()
defer per.Mu.Unlock()
err = dbs.Close()
if err != nil {
return
}
PanicOn(dbs.DeleteDBPath())
delete(per.Flatmap, flatkey{index: dbs.Index, shard: dbs.Shard})
// delete from the heirarchy
delete(dbs.ParentDBIndex.Shard, dbs.Shard)
return nil
return dbs, nil
}
func (per *DBPerShard) Close() (err error) {
@ -718,28 +480,7 @@ func (per *DBPerShard) Close() (err error) {
// If requireData, we open the database and see that it has a key, rather
// than assume that the database file presence is enough.
func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requireData bool) (map[uint64]bool, error) {
n := len(f.types)
if n != 1 && n != 2 {
PanicOn(fmt.Sprintf("internal error. only green or blue/green supported. we see types len %v", n))
}
var shards []map[uint64]bool
for _, ty := range f.types {
ss, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(ty, idx, roaringViewPath, requireData)
if err != nil {
return nil, err
}
shards = append(shards, ss)
}
// Note: we don't actually know when the blue call and when the green call comes
// through here. So if we are deleting a shard, we will see a difference earlier
// in one than the other. TestAPI_ClearFlagForImportAndImportValues for example.
// Hence we cannot do a blue-green check here for matching shards.
// If we are populating blue from green, it does matter that we return green.
return shards[n-1], nil
return f.dbPerShard.TypedDBPerShardGetShardsForIndex(f.typ, idx, roaringViewPath, requireData)
}
// if roaringViewPath is "" then for ty == roaringTxn we go to disk to discover
@ -751,10 +492,6 @@ func (f *TxFactory) GetShardsForIndex(idx *Index, roaringViewPath string, requir
// when a new DBShard is made, we will update the list of shards then. Thus
// the per.index2shard should always be up to date AFTER the first call here.
//
// Note: we cannot here call GetView2ShardsMapForIndex() because that only ever
// returns the green data and we are used during migration for both blue
// and green.
//
func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, roaringViewPath string, requireData bool) (shardMap map[uint64]bool, err error) {
// use the cache, always
@ -769,13 +506,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r
return shardMap, nil
}
i2ss, ok := per.index2shards[ty]
if !ok {
// index -> shardSet
i2ss = make(map[string]*shardSet)
per.index2shards[ty] = i2ss
}
// INVAR: i2ss is good, but may be an empty map
i2ss := per.index2shards
ss, ok := i2ss[idx.name]
if ok {
@ -785,7 +516,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r
// gotta read shards from disk directory layout.
setOfShards := newShardSet()
per.index2shards[ty][idx.name] = setOfShards
per.index2shards[idx.name] = setOfShards
// Upon return, cache the setOfShards value and reuse it next time
@ -854,13 +585,7 @@ func (per *DBPerShard) TypedDBPerShardGetShardsForIndex(ty txtype, idx *Index, r
}
func (per *DBPerShard) unprotectedTypedIndexShardHasData(ty txtype, idx *Index, shard uint64) (hasData bool, err error) {
whichty := 0
if len(per.types) == 2 {
if ty == per.types[1] {
whichty = 1
}
}
if ty != per.types[whichty] {
if ty != per.typ {
return
}
@ -871,7 +596,7 @@ func (per *DBPerShard) unprotectedTypedIndexShardHasData(ty txtype, idx *Index,
"per.GetDBShard(index='%v', shard='%v', ty='%v')", idx.name, shard, ty.String()))
}
return dbs.W[whichty].HasData()
return dbs.W.HasData()
}
func listDirUnderDir(root string, includeRoot bool, ignoreEmpty bool) (files []string, err error) {
@ -906,224 +631,6 @@ func listDirUnderDir(root string, includeRoot bool, ignoreEmpty bool) (files []s
return
}
// populateBlueFromGreen prepares for a blue_green run at startup time.
//
// It is called at the end of Holder.Open(). This allows the application
// of blue-green checking to pilosa instances that
// were previously run only with a single (solo) backend.
//
// PRE: This operation requires, at its start, either:
//
// (1) an empty blue database -- this allows transitioning from
// a solo database to blue_green checking where the solo
// becomes the green; or
//
// (2) that the blue data, if present, be logically
// identical to the green data -- this allows one to restart
// a pilosa that was already running in blue_green mode
// and remain in blue_green mode.
//
// In either case, the goal to to finish populateBlueFromGreen()
// and have the exact same logical set of data in both backends.
//
// Why must the data be identical after Holder.Open() finishes?
// Otherwise subsequent blue-green checks have no hope of
// being accurate.
//
// The blue is the destination -- this is always types[0].
// The green source is always types[1]. The mnemonic is blue_geen.
// The blue is first, so it is in types[0]. The green
// is second, in types[1]. For example, with PILOSA_STORAGE_BACKEND=bolt_roaring
// we have bolt as blue, and roaring as green. The contents of
// bolt must be empty or exactly match roaring. If bolt
// starts empty, it will be populated from roaring by
// populateBlueFromGreen().
//
func (dbs *DBShard) populateBlueFromGreen() (err error) {
n := len(dbs.W)
if n != 2 {
PanicOn(fmt.Sprintf("populateBlueFromGreen did not find 2 open DBs: have %v", n))
}
dest := dbs.W[0] // blue
src := dbs.W[1] // green
// copy all the key/container pairs.
// Since a shard is fairly small, we think one Tx will suffice.
readtx, err := src.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard})
PanicOn(err)
defer readtx.Rollback()
writetx, err := dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard})
PanicOn(err)
defer writetx.Rollback()
ctWriteCount := 0
for _, fld := range dbs.idx.Fields() {
field := fld.Name()
for _, vw := range fld.views() {
view := vw.name
citer, _, err := readtx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0)
if err != nil {
// might be an empty fragment. If so, let's not freak out.
if strings.Contains(err.Error(), "fragment not found") {
continue
} else {
writetx.Rollback()
return errors.Wrap(err, "DBShard.populateBlueFromGreen readtx.ContainerIterator")
}
}
for citer.Next() {
ckey, rc := citer.Value()
err := writetx.PutContainer(dbs.Index, field, view, dbs.Shard, ckey, rc)
if err != nil {
citer.Close()
writetx.Rollback()
return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.PutContainer")
}
ctWriteCount++
if ctWriteCount%1000 == 1 {
// regularly commiting smaller batches and the first batch as soon as
// possible massively speeds up writing to bolt.
//
// reference: https://github.com/boltdb/bolt/issues/94
//
// benbjohnson commented on Mar 25, 2014
// "Bulk loading more than 1000 items at a time is very slow. This is because nodes
// are not splitting before commit which causes large memmove() operations during insertion."
// runtime.memmove is taking all of the time in our pprof profile, when copying rbf to bolt, so we suspect it is this.
//
err = writetx.Commit()
if err != nil {
citer.Close()
writetx.Rollback()
return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.Commit")
}
writetx, err = dest.NewTx(writable, dbs.Index, Txo{Write: writable, Index: dbs.idx, Shard: dbs.Shard})
if err != nil {
citer.Close()
writetx.Rollback()
return errors.Wrap(err, "DBShard.populateBlueFromGreen writetx.NewTx inside citer.Next() loop")
}
}
}
citer.Close()
}
}
err = writetx.Commit()
if err != nil {
return errors.Wrap(err, "writetx.Commit()")
}
return nil
}
// verifyBlueEqualsGreen checks that blue and green are identical.
func (dbs *DBShard) verifyBlueEqualsGreen() (err error) {
n := len(dbs.W)
if n != 2 {
PanicOn(fmt.Sprintf("verifyBlueEqualsGreen did not find 2 open DBs: have %v", n))
}
blue := dbs.W[0]
green := dbs.W[1]
greentx, err := green.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard})
PanicOn(err)
defer greentx.Rollback()
bluetx, err := blue.NewTx(!writable, dbs.Index, Txo{Write: !writable, Index: dbs.idx, Shard: dbs.Shard})
PanicOn(err)
defer bluetx.Rollback()
for _, fld := range dbs.idx.Fields() {
field := fld.Name()
for _, vw := range fld.views() {
view := vw.name
gCiter, _, err := greentx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0)
if err != nil {
if strings.Contains(err.Error(), "fragment not found") {
continue
} else {
return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen greentx.ContainerIterator")
}
}
bCiter, _, err := bluetx.ContainerIterator(dbs.Index, field, view, dbs.Shard, 0)
if err != nil {
gCiter.Close()
if bCiter != nil {
bCiter.Close()
}
return errors.Wrap(err, "DBShard.verifyBlueEqualsGreen bluetx.ContainerIterator")
}
for gCiter.Next() {
greenCkey, greenc := gCiter.Value()
if !bCiter.Next() {
bCiter.Close()
gCiter.Close()
return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+
"sees missing blue container at index: '%v' field: '%v' view: '%v' "+
"shard: '%v' the greenCkey: '%v'",
dbs.Index, field, view, dbs.Shard, greenCkey))
}
blueCkey, bluec := bCiter.Value()
if blueCkey != greenCkey {
bCiter.Close()
gCiter.Close()
return fmt.Errorf("DBShard.verifyBlueEqualsGreen sees sequence-of-ckey "+
"difference: blueCkey %v not equal to greenCkey %v at index: '%v' field: '%v' view: '%v' "+
"shard: '%v'",
blueCkey, greenCkey, dbs.Index, field, view, dbs.Shard)
}
nGreen := greenc.N()
nBlue := bluec.N()
if nBlue != nGreen {
bCiter.Close()
gCiter.Close()
return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+
"sees variation in blue at index: '%v' field: '%v' view: '%v' "+
"shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v",
dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue))
}
err = bluec.BitwiseCompare(greenc)
if err != nil {
bCiter.Close()
gCiter.Close()
return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+
"sees variation in blue at index: '%v' field: '%v' view: '%v' "+
"shard: '%v' ckey: '%v' nHotGreen= %v nHotBlue= %v ; BitwiseCompare response: '%v'",
dbs.Index, field, view, dbs.Shard, greenCkey, nGreen, nBlue, err))
}
}
if bCiter.Next() {
blueCkey, _ := bCiter.Value()
bCiter.Close()
gCiter.Close()
return errors.Wrap(err, fmt.Sprintf("DBShard.verifyBlueEqualsGreen "+
"sees extra blue container (not present in green) at index: '%v' field: '%v' view: '%v' "+
"shard: '%v' the ckey: '%v'",
dbs.Index, field, view, dbs.Shard, blueCkey))
}
bCiter.Close()
gCiter.Close()
}
}
return nil
}
type FieldView2Shards struct {
// field -> view -> *shardSet
m map[string]map[string]*shardSet
@ -1232,15 +739,8 @@ func (vs *FieldView2Shards) removeField(name string) {
delete(vs.m, name)
}
// Note: cannot call this during migration, because
// it only ever returns the green shards if we are in blue-green.
func (per *DBPerShard) GetFieldView2ShardsMapForIndex(idx *Index) (vs *FieldView2Shards, err error) {
// for blue-green, it does matter that we return green, so we can migrate from it.
ty := per.types[0]
if per.isBlueGreen {
ty = per.types[1]
}
ty := per.typ
switch ty {
case roaringTxn:

22
dbshard_internal_test.go
View file

@ -84,7 +84,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
v2s.addViewShardSet(txkey.FieldView{Field: field, View: "standard"}, stdShardSet)
}
for _, src := range []string{"roaring", "bolt", "rbf"} {
for _, src := range []string{"roaring", "rbf"} {
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = src
holder := NewHolder(tmpdir, cfg)
@ -145,7 +145,7 @@ func Test_DBPerShard_GetShardsForIndex_LocalOnly(t *testing.T) {
tx.Rollback()
}
} else {
// non-roaring: rbf, bolt
// non-roaring: rbf
for _, shard := range []uint64{93, 223, 221, 215, 219, 217} {
tx := idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Shard: shard})
@ -191,14 +191,6 @@ rick/fields/_exists/views/standard/fragments/217
rick/fields/_exists/views/standard/fragments/93
rick/fields/_exists/views/standard/fragments/219
rick/fields/_exists/views/standard/fragments/223
`,
"bolt": `
rick/backends/backend-boltdb/shard.0093-bolt/bolt.db
rick/backends/backend-boltdb/shard.0215-bolt/bolt.db
rick/backends/backend-boltdb/shard.0217-bolt/bolt.db
rick/backends/backend-boltdb/shard.0219-bolt/bolt.db
rick/backends/backend-boltdb/shard.0221-bolt/bolt.db
rick/backends/backend-boltdb/shard.0223-bolt/bolt.db
`,
"rbf": `
rick/backends/backend-rbf/shard.0093-rbf
@ -225,7 +217,7 @@ func makeSampleRoaringDir(t *testing.T, root, index, backend string, minBytes in
}
var shard uint64
switch backend {
case "bolt", "rbf":
case "rbf":
shard = shards[i]
idx = helperCreateDBShard(h, index, shard)
@ -266,16 +258,8 @@ func helperCreateDBShard(h *Holder, index string, shard uint64) *Index {
}
// keep the ocd linter happy
var _ = makeBolttestDB
var _ = makeRBFtestDB
func makeBolttestDB(path string, h *Holder, shard uint64) {
i := uint64(1)
w, _ := mustOpenEmptyBoltWrapper(path)
BoltMustSetBitvalue(w, "index", "field", "view", shard, i)
w.Close()
}
func makeRBFtestDB(path string, h *Holder, shard uint64) {
i := uint64(1)

1
delete_test.go
View file

@ -27,7 +27,6 @@ import (
)
func TestExecutor_DeleteRecords(t *testing.T) {
pilosa.NotBlueGreenTest(t)
indexName := "i"
setup := func(t *testing.T, r *require.Assertions, c *test.Cluster) {
t.Helper()

1
executor_test.go
View file

@ -3843,7 +3843,6 @@ func TestExecutor_Execute_Existence(t *testing.T) {
hldr2 := c.GetHolder(0)
index2 := hldr2.Index("i")
_ = index2
//index2.Dump("after reopen")
if res, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Not(Row(f=10))`}); err != nil {
t.Fatal(err)

91
fragment.go
View file

@ -47,6 +47,7 @@ import (
"github.com/molecula/featurebase/v2/roaring"
"github.com/molecula/featurebase/v2/shardwidth"
"github.com/molecula/featurebase/v2/stats"
"github.com/molecula/featurebase/v2/storage"
"github.com/molecula/featurebase/v2/testhook"
"github.com/molecula/featurebase/v2/topology"
"github.com/molecula/featurebase/v2/tracing"
@ -75,9 +76,6 @@ const (
// snapshotExt is the file extension used for an in-process snapshot.
snapshotExt = ".snapshotting"
// copyExt is the file extension used for the temp file used while copying.
copyExt = ".copying"
// cacheExt is the file extension for persisted cache ids.
cacheExt = ".cache"
@ -441,7 +439,7 @@ func (f *fragment) inspectStorage(data []byte, file *os.File, newGen generation,
// (remapping an existing bitmap to match a new backing store).
func (f *fragment) openStorage(unmarshalData bool) error {
useRowCache := f.idx.Txf().UseRowCache()
useRowCache := storage.RowCacheEnabled()
if !f.idx.NeedsSnapshot() {
f.gen = &NopGeneration{}
if useRowCache {
@ -626,7 +624,7 @@ func (f *fragment) mustRow(tx Tx, rowID uint64) *Row {
// (updating the cache).
func (f *fragment) unprotectedRow(tx Tx, rowID uint64) (*Row, error) {
useRowCache := tx.UseRowCache()
useRowCache := storage.RowCacheEnabled()
if useRowCache {
if f.rowCache == nil {
f.rowCache = newSimpleCache()
@ -699,7 +697,7 @@ func (f *fragment) setBit(tx Tx, rowID, columnID uint64) (changed bool, err erro
if tx.Type() == RoaringTxn {
return changed, errors.New("internal error: f.gen was nil and tx.Type is RoaringTxn - should never happen under roaring b/c storage should be open")
}
// else blue green or transactional backend. Just do it.
// else transactional backend. Just do it.
err = doSetFunc()
}
return changed, err
@ -744,7 +742,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo
delete(f.checksums, int(rowID/HashBlockSize))
// Increment number of operations until snapshot is required.
tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, 1)
f.incrementOpN(1)
// If we're using a cache, update it. Otherwise skip the
// possibly-expensive count operation.
@ -757,7 +755,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo
}
// Drop the rowCache entry; it's wrong, and we don't want to force
// a new copy if no one's reading it.
if tx.UseRowCache() && f.rowCache != nil {
if storage.RowCacheEnabled() && f.rowCache != nil {
f.rowCache.Add(rowID, nil)
}
@ -809,7 +807,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b
delete(f.checksums, int(rowID/HashBlockSize))
// Increment number of operations until snapshot is required.
tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, 1)
f.incrementOpN(1)
// If we're using a cache, update it. Otherwise skip the
// possibly-expensive count operation.
@ -822,7 +820,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b
}
// Drop the rowCache entry; it's wrong, and we don't want to force
// a new copy if no one's reading it.
if tx.UseRowCache() && f.rowCache != nil {
if storage.RowCacheEnabled() && f.rowCache != nil {
f.rowCache.Add(rowID, nil)
}
@ -891,7 +889,7 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo
}
// invalidate rowCache for this row.
if tx.UseRowCache() && f.rowCache != nil {
if storage.RowCacheEnabled() && f.rowCache != nil {
f.rowCache.Add(rowID, nil)
}
@ -942,7 +940,7 @@ func (f *fragment) unprotectedClearRow(tx Tx, rowID uint64) (changed bool, err e
// Clear the row in cache.
f.cache.Add(rowID, 0)
if tx.UseRowCache() && f.rowCache != nil {
if storage.RowCacheEnabled() && f.rowCache != nil {
f.rowCache.Add(rowID, nil)
}
@ -2426,7 +2424,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64
if f.storage != nil {
wp = &f.storage.OpWriter
}
useRowCache := tx.UseRowCache()
useRowCache := storage.RowCacheEnabled()
doFunc := func() error {
if len(set) > 0 {
@ -2438,7 +2436,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64
return errors.Wrap(err, "adding positions")
}
f.stats.Count(MetricImportedN, int64(changedN), 1)
tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, changedN)
f.incrementOpN(changedN)
}
if len(clear) > 0 {
@ -2448,7 +2446,7 @@ func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64
return errors.Wrap(err, "clearing positions")
}
f.stats.Count(MetricClearedN, int64(changedN), 1)
tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, changedN)
f.incrementOpN(changedN)
}
// Update cache counts for all affected rows.
@ -2735,7 +2733,7 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b
rowSize := uint64(1 << shardVsContainerExponent)
span, ctx := tracing.StartSpanFromContext(ctx, "importRoaring.ImportRoaringBits")
useRowCache := tx.UseRowCache()
useRowCache := storage.RowCacheEnabled()
var changed int
var rowSet map[uint64]int
var wp *io.Writer
@ -2749,7 +2747,7 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b
return err
}
changed, rowSet, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, rit, clear, true, rowSize, nil)
changed, rowSet, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, rit, clear, true, rowSize)
return err
})
@ -2789,7 +2787,7 @@ func (f *fragment) unprotectedImportRoaring(ctx context.Context, tx Tx, data []b
span, _ = tracing.StartSpanFromContext(ctx, "importRoaring.incrementOpN")
tx.IncrementOpN(f.index(), f.field(), f.view(), f.shard, changed)
f.incrementOpN(changed)
span.Finish()
return nil
@ -2815,6 +2813,10 @@ func (f *fragment) incrementOpN(changed int) {
if changed <= 0 {
return
}
// don't count opN or ops if our index doesn't want snapshots
if !f.idx.NeedsSnapshot() {
return
}
f.opN += changed
f.ops++
if f.opN > f.MaxOpN {
@ -3015,11 +3017,15 @@ func (f *fragment) writeStorageToArchive(tw *tar.Writer) error {
tx := f.idx.holder.txf.NewTx(Txo{Write: !writable, Index: f.idx, Shard: f.shard})
defer tx.Rollback()
file, sz, err := tx.RoaringBitmapReader(f.index(), f.field(), f.view(), f.shard, f.path())
rbm, err := tx.RoaringBitmap(f.index(), f.field(), f.view(), f.shard)
if err != nil {
return err
return errors.Wrap(err, "RoaringBitmapReader RoaringBitmap")
}
var buf bytes.Buffer
sz, err := rbm.WriteTo(&buf)
if err != nil {
return errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)")
}
defer file.Close()
// Write archive header.
if err := tw.WriteHeader(&tar.Header{
@ -3033,7 +3039,7 @@ func (f *fragment) writeStorageToArchive(tw *tar.Writer) error {
// Copy the file up to the last known size.
// This is done outside the lock because the storage format is append-only.
if _, err := io.CopyN(tw, file, sz); err != nil {
if _, err := io.CopyN(tw, &buf, sz); err != nil {
return errors.Wrap(err, "copying")
}
return nil
@ -3086,8 +3092,7 @@ func (f *fragment) ReadFrom(r io.Reader) (n int64, err error) {
// Process file based on file name.
switch hdr.Name {
case "data":
idx := f.holder.Index(f.index())
tx := idx.holder.txf.NewTx(Txo{Write: writable, Index: idx, Fragment: f, Shard: f.shard})
tx := f.holder.txf.NewTx(Txo{Write: writable, Index: f.idx, Fragment: f, Shard: f.shard})
defer tx.Rollback()
if err := f.fillFragmentFromArchive(tx, tr); err != nil {
return 0, errors.Wrap(err, "reading storage")
@ -3131,7 +3136,7 @@ func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error {
if err != nil {
return errors.Wrap(err, "fillFragmentFromArchive NewRoaringIterator")
}
changed, rowSet, err := tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, clear, log, rowSize, data)
changed, rowSet, err := tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, clear, log, rowSize)
_, _ = changed, rowSet
if err != nil {
return errors.Wrap(err, "fillFragmentFromArchive ImportRoaringBits")
@ -3139,40 +3144,6 @@ func (f *fragment) fillFragmentFromArchive(tx Tx, r io.Reader) error {
return nil
}
func (f *fragment) readStorageFromArchive(r io.Reader) error {
// Create a temporary file to copy into.
path := f.path() + copyExt
file, err := os.Create(path)
if err != nil {
return errors.Wrap(err, "creating directory")
}
defer file.Close()
// Copy reader into temporary path.
if _, err = io.Copy(file, r); err != nil {
return errors.Wrap(err, "copying")
}
// TODO(jea): isn't this next Rename a file handle leak?
// try closing first
if err := f.closeStorage(); err != nil {
return errors.Wrap(err, "closeStorage-prior-to-Rename-and-openStorage")
}
// Move snapshot to data file location.
if err := os.Rename(path, f.path()); err != nil {
return errors.Wrap(err, "renaming")
}
// Reopen storage.
if err := f.openStorage(true); err != nil {
return errors.Wrap(err, "opening")
}
return nil
}
func (f *fragment) readCacheFromArchive(r io.Reader) error {
// Slurp data from reader and write to disk.
buf, err := ioutil.ReadAll(r)
@ -3369,7 +3340,7 @@ func (f *fragment) intRowIterator(tx Tx, wrap bool, filters ...roaring.BitmapFil
// accumulator [column ID] -> [int value]
acc := make(map[uint64]int64)
if tx.UseRowCache() {
if storage.RowCacheEnabled() {
// needs a write lock since it will update the f.rowCache
f.mu.Lock()
defer f.mu.Unlock()

43
fragment_internal_test.go
View file

@ -183,7 +183,6 @@ func TestFragment_RowcacheMap(t *testing.T) {
// Ensure a fragment can clear a row.
func TestFragment_ClearRow(t *testing.T) {
NotBlueGreenTest(t)
f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "")
_ = idx
defer f.Clean(t)
@ -215,7 +214,6 @@ func TestFragment_ClearRow(t *testing.T) {
// Ensure a fragment can set a row.
func TestFragment_SetRow(t *testing.T) {
NotBlueGreenTest(t)
f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 7, "")
_ = idx
defer f.Clean(t)
@ -1728,7 +1726,7 @@ func roaringOnlyBenchmark(b *testing.B) {
// Ensure a fragment can be copied to another fragment.
func TestFragment_WriteTo_ReadFrom(t *testing.T) {
roaringOnlyTest(t)
// roaringOnlyTest(t)
f0, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "")
defer f0.Clean(t)
@ -1741,6 +1739,10 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
} else if _, err := f0.clearBit(tx, 1000, 1); err != nil {
t.Fatal(err)
}
err := tx.Commit()
if err != nil {
t.Fatalf("committing write: %v", err)
}
// Verify cache is populated.
if n := f0.cache.Len(); n != 1 {
@ -1755,7 +1757,9 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
}
// Read into another fragment.
f1, _, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "")
f1, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, "")
tx.Rollback()
defer f1.Clean(t)
if rn, err := f1.ReadFrom(&buf); err != nil { // eventually calls fragment.fillFragmentFromArchive
@ -1763,6 +1767,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
} else if wn != rn {
t.Fatalf("read/write byte count mismatch: wn=%d, rn=%d", wn, rn)
}
// make a read-only Tx after ReadFrom has committed.
tx = idx.holder.txf.NewTx(Txo{Write: !writable, Index: idx, Fragment: f1, Shard: f1.shard})
// Verify cache is in other fragment.
if n := f1.cache.Len(); n != 1 {
@ -3049,7 +3055,7 @@ func BenchmarkImportRoaringUpdate(b *testing.B) {
// to generate an op log and/or snapshot.
itr, err := roaring.NewRoaringIterator(data)
PanicOn(err)
_, _, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, false, false, 0, nil)
_, _, err = tx.ImportRoaringBits(f.index(), f.field(), f.view(), f.shard, itr, false, false, 0)
if err != nil {
b.Errorf("import error: %v", err)
}
@ -5215,15 +5221,13 @@ func TestImportValueConcurrent(t *testing.T) {
// we will be making a new Tx each time, so we can rollback the default provided one.
tx.Rollback()
types := idx.holder.txf.TxTypes()
for _, ty := range types {
switch ty {
case roaringTxn:
t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " +
"blueGreenTx because the lack of transactional consistency " +
"from Roaring-per-file will create false comparison " +
"failures."))
}
ty := idx.holder.txf.TxTyp()
switch ty {
case roaringTxn:
t.Skip(fmt.Sprintf("skipping TestImportValueConcurrent under " +
"roaring because the lack of transactional consistency " +
"from Roaring-per-file will create false comparison " +
"failures."))
}
eg := &errgroup.Group{}
@ -5364,11 +5368,6 @@ func TestImportValueRowCache(t *testing.T) {
// do we see races/corruption around concurrent read/write.
// especially on writes to the row cache.
func TestFragmentConcurrentReadWrite(t *testing.T) {
// actual transaction backends, there won't be any
// data, and in particular, the blue-green tests will
// note this and fire a false-positive.
NotBlueGreenTest(t)
f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, 0, CacheTypeRanked)
defer f.Clean(t)
tx.Rollback()
@ -5528,12 +5527,6 @@ func TestFragment_Bug_Q2DoubleDelete(t *testing.T) {
}
}
func NotBlueGreenTest(t *testing.T) {
if strings.Contains(CurrentBackend(), "_") {
t.Skip("skip under blue green")
}
}
var mutexSamplesPrepared sync.Once
func requireMutexSampleData(tb testing.TB) {

2
go.mod
View file

@ -16,8 +16,6 @@ require (
github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 // indirect
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311
github.com/go-test/deep v1.0.7
github.com/gogo/protobuf v1.3.2
github.com/golang/protobuf v1.3.3

4
go.sum
View file

@ -86,10 +86,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8=
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311 h1:AAXH0ZvYIHHqU06ASy0H2tYAkAGrQlZvEy2QZrrtt4E=
github.com/glycerine/idem v0.0.0-20190127113923-7a8083893311/go.mod h1:B72P/ZM99sNiCmaQJflpmMAF5LsDzStpLdWzn0+Vr2Y=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=

19
holder.go
View file

@ -302,7 +302,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
txf, err := NewTxFactory(cfg.StorageConfig.Backend, h.IndexesPath(), h)
PanicOn(err)
h.txf = txf
h.txf.blueGreenOffIfRunningBlueGreen()
_ = testhook.Created(h.Auditor, h, nil)
return h
@ -605,8 +604,6 @@ func (h *Holder) Open() error {
h.txf = txf
}
h.txf.blueGreenOffIfRunningBlueGreen()
// Reset closing in case Holder is being reopened.
h.closing = make(chan struct{})
@ -713,13 +710,6 @@ func (h *Holder) Open() error {
return errors.Wrap(err, "Holder.Open h.txf.Open()")
}
// under blue_green, we must sync blue from green before we turn on checking.
if err := h.txf.green2blue(h); err != nil {
return errors.Wrap(err, "Holder.Open h.txf.green2blue(h)")
}
h.txf.blueGreenOnIfRunningBlueGreen()
if h.cfg.LookupDBDSN != "" {
h.Logger.Printf("connecting to lookup database")
@ -814,9 +804,6 @@ func (h *Holder) Close() error {
if globalUseStatTx {
fmt.Printf("%v\n", globalCallStats.report())
}
if h.txf != nil && h.txf.blueGreenReg != nil {
h.txf.blueGreenReg.Close()
}
h.Stats.Close()
@ -2131,12 +2118,6 @@ func (h *Holder) addIndex(idx *Index) {
h.imu.Unlock()
}
func (h *Holder) DumpAllShards() {
h.mu.RLock()
defer h.mu.RUnlock()
h.txf.dbPerShard.DumpAll()
}
func (h *Holder) Txf() *TxFactory {
h.mu.Lock()
defer h.mu.Unlock()

68
holder_internal_test.go
View file

@ -22,7 +22,6 @@ import (
"github.com/molecula/featurebase/v2/disco"
"github.com/molecula/featurebase/v2/testhook"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
)
var _ = fmt.Printf
@ -109,73 +108,6 @@ func testSetBit(t *testing.T, h *Holder, index, field string, rowID, columnID ui
}
}
func testMustHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) {
//shard := columnID / ShardWidth
// hmm... if its a new holder, meta data isn't there, so ask for it.
idx, err := h.CreateIndexIfNotExists(index, IndexOptions{})
PanicOn(err)
f := idx.Field(field)
if f == nil {
t.Fatalf("no such field '%v'", field)
}
row, err := f.Row(nil, rowID)
if err != nil {
t.Fatalf("error getting field.Row(rowID=%v): %v", rowID, err)
}
cols := row.Columns()
if len(cols) == 0 {
t.Fatalf("error getting field.Row().Columns(): empty columns, colID %v bit was not hot", columnID)
}
for _, c := range cols {
if c == columnID {
return // ok, found it.
}
}
t.Fatalf("error getting field.Row().Columns(): colID %v bit was not hot", columnID)
}
func testMustNotHaveBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) {
if testHasBit(t, h, index, field, rowID, columnID) {
t.Fatalf("error, expected no bit but this bit was hot: index='%v', field='%v', rowID='%v', columnID='%v'", index, field, rowID, columnID)
}
}
func testHasBit(t *testing.T, h *Holder, index, field string, rowID, columnID uint64) bool {
idx := h.Index(index)
if idx == nil {
return false // not even an index by this name. Obviously no hot bits either.
}
f := idx.Field(field)
if f == nil {
return false
}
row, err := f.Row(nil, rowID)
if err != nil {
return false
}
cols := row.Columns()
if len(cols) == 0 {
return false
}
for _, c := range cols {
if c == columnID {
return true // ok, found it.
}
}
return false
}
func TestHolderOperatorProcess(t *testing.T) {
h, path, err := makeHolder(t, "")
if err != nil {

10
index.go
View file

@ -27,7 +27,6 @@ import (
"github.com/molecula/featurebase/v2/roaring"
"github.com/molecula/featurebase/v2/stats"
"github.com/molecula/featurebase/v2/testhook"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
@ -464,7 +463,6 @@ func (i *Index) Close() error {
// make it clear what the Index.AvailableShards() calls are trying to obtain.
const includeRemote = false
const localOnly = true
// AvailableShards returns a bitmap of all shards with data in the index.
func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap {
@ -853,14 +851,6 @@ func FormatQualifiedIndexName(index string) string {
return fmt.Sprintf("%s\x00", index)
}
// Dump prints to stdout the contents of the roaring Containers
// stored in idx. Mostly for debugging.
func (i *Index) Dump(label string) {
fileline := FileLine(2)
fmt.Printf("\n%v Dump: %v\n\n", fileline, label)
i.holder.txf.dbPerShard.DumpAll()
}
func (i *Index) Txf() *TxFactory {
return i.holder.txf
}

115
pjobs.go
View file

@ -1,115 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"sync"
"github.com/glycerine/idem"
)
// parallelJobs runs functions in parallel on a goroutine
// pool that has nGoro goroutines.
type parallelJobs struct {
nGoro int
jobQ chan func(worker int) error
halters []*idem.Halter
// err is protected by errmu
err error
errmu sync.Mutex
}
func newParallelJobs(nGoro int) (p *parallelJobs) {
if nGoro < 1 {
// 0 really means,
// "turn it up to 11".
// same for negative.
nGoro = 10000
}
// maximum 10K goroutines
if nGoro > 10000 {
nGoro = 10000
}
p = &parallelJobs{
nGoro: nGoro,
jobQ: make(chan func(worker int) error, 10000),
halters: make([]*idem.Halter, nGoro),
}
for j := 0; j < nGoro; j++ {
h := idem.NewHalter()
p.halters[j] = h
}
for i, h := range p.halters {
go func(h *idem.Halter, worker int) {
defer h.MarkDone()
for {
select {
case <-h.ReqStop.Chan:
return
case f, ok := <-p.jobQ:
if !ok {
// channel closed, finish up
return
}
err1 := f(worker)
if err1 != nil {
p.errmu.Lock()
if p.err == nil {
p.err = err1
}
p.errmu.Unlock()
// an error occurred, tell everyone to stop
for _, h2 := range p.halters {
h2.RequestStop()
}
return
}
}
}
}(h, i)
}
return
}
// return value accepted will be false if we are shutting down
// due to an error.
func (p *parallelJobs) run(fun func(worker int) error) (accepted bool) {
select {
case <-p.halters[0].ReqStop.Chan:
return false
case p.jobQ <- fun:
return true
}
}
func (p *parallelJobs) waitForFinish() error {
// tell the workers no more jobs.
close(p.jobQ)
// wait for everyone to finish
for i, h := range p.halters {
_ = i
<-h.Done.Chan
}
return p.err
}

51
pjobs_test.go
View file

@ -1,51 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pilosa
import (
"fmt"
"sync/atomic"
"testing"
)
func Test_ParallelJobs_EarlyShutdown_WaitsForAllGoro(t *testing.T) {
const n = 10000 // total jobs to run
var errLastOne = fmt.Errorf("the last job has run, and returned this error")
pj := newParallelJobs(100)
nTotal := int64(0)
for i := 0; i < n; i++ {
accepted := pj.run(func(worker int) error {
highpoint := atomic.AddInt64(&nTotal, 1)
switch int(highpoint) {
case n - 1:
return errLastOne
}
return nil
})
if !accepted {
panic("should have been accepted")
}
}
err := pj.waitForFinish()
tot := atomic.LoadInt64(&nTotal)
if int(tot) != n {
panic(fmt.Sprintf("We didn't run them all? tot=%v, n=%v; pj.jobQ len %v; err='%v'", tot, n, len(pj.jobQ), err))
}
if err != errLastOne {
panic("expected to see errLastOne")
}
// good: finished cleanly.
}

94
rbf.go
View file

@ -15,15 +15,12 @@
package pilosa
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"math"
"os"
"strings"
"sync"
"sync/atomic"
"github.com/molecula/featurebase/v2/rbf"
rbfcfg "github.com/molecula/featurebase/v2/rbf/cfg"
@ -76,8 +73,6 @@ func (w *RbfDBWrapper) CleanupTx(tx Tx) {
w.muDb.Lock()
delete(w.openTx, r)
//vv("rbf CleanupTx gid %v about to call r.o.dbs.Cleanup(tx.Sn=%v)", curGID(), tx.Sn())
r.o.dbs.Cleanup(tx) // release the read/write lock.
w.muDb.Unlock()
}
@ -186,20 +181,12 @@ type RBFTx struct {
initialIndex string
tx *rbf.Tx
o Txo
sn int64 // serial number
Db *RbfDBWrapper
done bool
mu sync.Mutex // protect done as it changes state
}
func (tx *RBFTx) IsDone() (done bool) {
tx.mu.Lock()
done = tx.done
tx.mu.Unlock()
return
}
func (tx *RBFTx) DBPath() string {
return tx.tx.DBPath()
}
@ -210,17 +197,13 @@ func (tx *RBFTx) Type() string {
func (tx *RBFTx) Rollback() {
tx.tx.Rollback()
// must happen after actual rollback
tx.Db.CleanupTx(tx)
}
func (tx *RBFTx) Commit() (err error) {
err = tx.tx.Commit()
// must happen after actual commit
tx.Db.CleanupTx(tx)
return
return err
}
func (tx *RBFTx) RoaringBitmap(index, field, view string, shard uint64) (*roaring.Bitmap, error) {
@ -412,10 +395,6 @@ func (tx *RBFTx) Min(index, field, view string, shard uint64) (uint64, bool, err
return tx.tx.Min(rbfName(index, field, view, shard))
}
func (tx *RBFTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
return tx.tx.UnionInPlace(rbfName(index, field, view, shard), others...)
}
// CountRange returns the count of hot bits in the start, end range on the fragment.
// roaring.countRange counts the number of bits set between [start, end).
func (tx *RBFTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
@ -426,24 +405,8 @@ func (tx *RBFTx) OffsetRange(index, field, view string, shard uint64, offset, st
return tx.tx.OffsetRange(rbfName(index, field, view, shard), offset, start, end)
}
func (tx *RBFTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {}
func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
return tx.tx.ImportRoaringBits(rbfName(index, field, view, shard), rit, clear, log, rowSize, data)
}
func (tx *RBFTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
rbm, err := tx.RoaringBitmap(index, field, view, shard)
if err != nil {
return nil, -1, errors.Wrap(err, "RoaringBitmapReader RoaringBitmap")
}
var buf bytes.Buffer
sz, err = rbm.WriteTo(&buf)
if err != nil {
return nil, -1, errors.Wrap(err, "RoaringBitmapReader rbm.WriteTo(buf)")
}
return ioutil.NopCloser(&buf), sz, err
func (tx *RBFTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
return tx.tx.ImportRoaringBits(rbfName(index, field, view, shard), rit, clear, log, rowSize)
}
func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
@ -452,39 +415,6 @@ func (tx *RBFTx) NewTxIterator(index, field, view string, shard uint64) *roaring
return b.Iterator()
}
func (tx *RBFTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
func (tx *RBFTx) Dump(short bool, shard uint64) {
tx.tx.Dump(short, shard)
}
// Readonly is true if the transaction is not read-and-write, but only doing reads.
func (tx *RBFTx) Readonly() bool {
return !tx.tx.Writable()
}
func (tx *RBFTx) Group() *TxGroup {
return tx.o.Group
}
func (tx *RBFTx) Options() Txo {
return tx.o
}
func (tx *RBFTx) Sn() int64 {
return tx.sn
}
func (tx *RBFTx) UseRowCache() bool {
// since RFB returns memory mapped data, we can't use
// the rowCache without first making a copy.
// So we only use the rowCache if the copy is
// enabled.
return storage.EnableRowCache()
}
func (tx *RBFTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
return tx.tx.ApplyFilter(rbfName(index, field, view, shard), ckey, filter)
}
@ -591,20 +521,16 @@ func (w *RbfDBWrapper) OpenDB() error {
return nil
}
var globalNextTxSnRBFTx int64
func (w *RbfDBWrapper) NewTx(write bool, initialIndex string, o Txo) (_ Tx, err error) {
tx, err := w.db.Begin(write)
if err != nil {
return nil, err
}
sn := atomic.AddInt64(&globalNextTxSnRBFTx, 1)
rtx := &RBFTx{
tx: tx,
initialIndex: initialIndex,
o: o,
sn: sn,
Db: w,
}
@ -629,20 +555,6 @@ func (w *RbfDBWrapper) DeleteFragment(index, field, view string, shard uint64, f
return tx.Commit()
}
func (w *RbfDBWrapper) DeleteDBPath(dbs *DBShard) error {
path := dbs.pathForType(rbfTxn)
return os.RemoveAll(path)
}
func (w *RbfDBWrapper) OpenListString() (r string) {
return "rbf OpenListString not implemented yet"
}
func (w *RbfDBWrapper) OpenSnList() (slc []int64) {
w.muDb.Lock()
for v := range w.openTx {
slc = append(slc, v.sn)
}
w.muDb.Unlock()
return
}

110
rbf/cursor_internal_test.go
View file

@ -59,20 +59,12 @@ func TestCursor_RoaringImport(t *testing.T) {
tx := MustBegin(t, db, true)
defer tx.Rollback()
changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize)
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
_ = rowSet
if changed != 1 {
t.Fatalf("expected 1 changed, got %v", changed)
}
if false {
cur, err := tx.cursor(name)
PanicOn(err)
cur.dump()
_ = cur.tx.dumpAllPages(true)
}
}
func TestCursor_RoaringImport_clear_bits(t *testing.T) {
@ -93,7 +85,7 @@ func TestCursor_RoaringImport_clear_bits(t *testing.T) {
tx := MustBegin(t, db, true)
defer tx.Rollback()
changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize)
PanicOn(err)
_ = rowSet
if changed != 1 {
@ -104,21 +96,12 @@ func TestCursor_RoaringImport_clear_bits(t *testing.T) {
clear = true
itr2 := getRoaringIter([]uint64{1}...)
changed, rowSet, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil)
changed, rowSet, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize)
PanicOn(err)
_ = rowSet
if changed != 1 {
t.Fatalf("expected 1 changed on clear true, got %v", changed)
}
if false {
cur, err := tx.cursor(name)
PanicOn(err)
cur.dump()
_ = cur.tx.dumpAllPages(true)
}
}
func TestCursor_RoaringImport_two_leaves(t *testing.T) {
@ -152,20 +135,12 @@ func TestCursor_RoaringImport_two_leaves(t *testing.T) {
tx := MustBegin(t, db, true)
defer tx.Rollback()
changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
changed, rowSet, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize)
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
_ = rowSet
if changed != 6000 {
t.Fatalf("expected 6000 bits changed, got %v", changed)
}
if false {
cur, err := tx.cursor(name)
PanicOn(err)
cur.dump()
_ = cur.tx.dumpAllPages(true)
}
}
func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) {
@ -211,7 +186,7 @@ func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) {
defer tx.Rollback()
//vv("DONE WITH Add()")
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize)
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != expectedBitsChanged-3000 {
t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged-3000, changed)
@ -219,37 +194,23 @@ func TestCursor_RoaringImport_many_leaves_manual_split(t *testing.T) {
//vv("changed on set is %v", changed)
//vv("about to do itr2, that starts with key %v", itr2.ContainerKeys()[0])
changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil)
changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize)
PanicOn(err)
if changed != 3000 {
t.Fatalf("expected %v bits changed, got %v", 3000, changed)
}
//vv("done with itr2")
dump := func() {
cur, err := tx.cursor(name)
PanicOn(err)
//cur.dump()
_ = cur.tx.dumpAllPages(true)
}
_ = dump
//dump()
//vv("now clear")
// now clear
clear = true
//itr3 := getRoaringIter(want[len(want)-3000:]...)
itr3 := getRoaringIter(want...)
changed, _, err = tx.ImportRoaringBits(name, itr3, clear, false, rowSize, nil)
changed, _, err = tx.ImportRoaringBits(name, itr3, clear, false, rowSize)
PanicOn(err)
if changed != expectedBitsChanged {
// cursor_internal_test.go:235: expected 2,724,000 bits changed, got 2,721,000
t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed)
}
//dump()
}
func TestCursor_RoaringImport_auto_many_leaves(t *testing.T) {
@ -292,34 +253,21 @@ func TestCursor_RoaringImport_auto_many_leaves(t *testing.T) {
defer tx.Rollback()
//vv("DONE WITH Add()")
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize)
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != expectedBitsChanged {
t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed)
}
//vv("changed on set is %v", changed)
dump := func() {
cur, err := tx.cursor(name)
PanicOn(err)
//cur.dump()
_ = cur.tx.dumpAllPages(true)
}
_ = dump
//dump()
//vv("now clear")
// now clear
clear = true
itr2 := getRoaringIter(want...)
changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize, nil)
changed, _, err = tx.ImportRoaringBits(name, itr2, clear, false, rowSize)
PanicOn(err)
if changed != expectedBitsChanged {
t.Fatalf("expected %v bits changed, got %v", expectedBitsChanged, changed)
}
//dump()
}
func TestCursor_putBranchCellsHandlesLotsOfNewBranchesAtTheRoot(t *testing.T) {
@ -430,8 +378,6 @@ func TestCursor_putBranchCellsHandlesLotsOfNewBranchesAtTheRoot(t *testing.T) {
err = c.putBranchCells(0, branches)
PanicOn(err)
//c.tx.dumpAllPages(true)
}
func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) {
@ -475,29 +421,18 @@ func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) {
tx := MustBegin(t, db, true)
defer tx.Rollback()
dump := func() {
cur, err := tx.cursor(name)
PanicOn(err)
//cur.dump()
_ = cur.tx.dumpAllPages(true)
}
_ = dump
for i := 0; i < NbranchCells; i++ {
itr := getRoaringIter(want[i*3000 : (i+1)*3000]...)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize)
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != 3000 {
t.Fatalf("expected %v bits changed, got %v", 3000, changed)
}
//vv("changed on set is %v", changed)
//dump()
}
//vv("now clear")
// now clear
clear = true
@ -505,13 +440,11 @@ func TestCursor_incrementally_add_pages_and_view_them(t *testing.T) {
itr := getRoaringIter(want[i*3000 : (i+1)*3000]...)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize)
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != 3000 {
t.Fatalf("expected %v bits changed, got %v", 3000, changed)
}
//vv("changed on set is %v", changed)
//dump()
}
}
@ -558,35 +491,22 @@ func TestCursor_from_B_to_C(t *testing.T) {
tx := MustBegin(t, db, true)
defer tx.Rollback()
dump := func() {
cur, err := tx.cursor(name)
PanicOn(err)
//cur.dump()
_ = cur.tx.dumpAllPages(true)
}
_ = dump
itr := getRoaringIter(want[:6000]...)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize)
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != 6000 {
t.Fatalf("expected %v bits changed, got %v", 6000, changed)
}
//vv("changed on set is %v", changed)
//dump()
//vv("STARTING TO ADD C")
itr = getRoaringIter(want[6000:9000]...)
changed, _, err = tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
changed, _, err = tx.ImportRoaringBits(name, itr, clear, false, rowSize)
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != 3000 {
t.Fatalf("expected %v bits changed, got %v", 3000, changed)
}
//vv("changed on set is %v", changed)
//dump()
//vv("now clear")
// now clear
clear = true
@ -595,12 +515,10 @@ func TestCursor_from_B_to_C(t *testing.T) {
itr := getRoaringIter(want[i*3000 : (i+1)*3000]...)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize, nil)
changed, _, err := tx.ImportRoaringBits(name, itr, clear, false, rowSize)
PanicOn(err) // writeTheTailOfLeafCellsFromIter has to be AFTER any pre-existing data. ckey=0 was found already in db.
if changed != 3000 {
t.Fatalf("expected %v bits changed, got %v", 3000, changed) // failing here got 0
}
//vv("changed on clear is %v", changed)
//dump()
}
}

8
rbf/cursorx.go
View file

@ -174,7 +174,7 @@ func intoContainer(l leafCell, tx *Tx, replacing *roaring.Container, target []by
orig := l.Data
var cpMaybe []byte
var mapped bool
if storage.EnableRowCache() || tx.db.cfg.DoAllocZero {
if storage.RowCacheEnabled() || tx.db.cfg.DoAllocZero {
// make a copy, otherwise the rowCache will see corrupted data
// or mmapped data that may disappear.
cpMaybe = target[:len(orig)]
@ -191,7 +191,7 @@ func intoContainer(l leafCell, tx *Tx, replacing *roaring.Container, target []by
case ContainerTypeBitmapPtr:
_, bm, _ := tx.leafCellBitmap(toPgno(cpMaybe))
cloneMaybe := bm
if storage.EnableRowCache() {
if storage.RowCacheEnabled() {
cloneMaybe = (*[1024]uint64)(unsafe.Pointer(&target[0]))[:1024]
copy(cloneMaybe, bm)
}
@ -217,7 +217,7 @@ func toContainer(l leafCell, tx *Tx) (c *roaring.Container) {
orig := l.Data
var cpMaybe []byte
var mapped bool
if storage.EnableRowCache() || tx.db.cfg.DoAllocZero {
if storage.RowCacheEnabled() || tx.db.cfg.DoAllocZero {
// make a copy, otherwise the rowCache will see corrupted data
// or mmapped data that may disappear.
cpMaybe = make([]byte, len(orig))
@ -234,7 +234,7 @@ func toContainer(l leafCell, tx *Tx) (c *roaring.Container) {
case ContainerTypeBitmapPtr:
_, bm, _ := tx.leafCellBitmap(toPgno(cpMaybe))
cloneMaybe := bm
if storage.EnableRowCache() {
if storage.RowCacheEnabled() {
cloneMaybe = make([]uint64, len(bm))
copy(cloneMaybe, bm)
}

2
rbf/db.go
View file

@ -319,7 +319,7 @@ func (db *DB) Close() (err error) {
// least a single hot bit inside the db
// in order to return hasAnyRecords true.
//
// HasData is used by backend migration and blue/green checks.
// HasData is used by backend migration.
//
// If there is a disk error we return (false, error), so always
// check the error before deciding if hasAnyRecords is valid.

147
rbf/tx.go
View file

@ -23,7 +23,6 @@ import (
"sync"
"github.com/benbjohnson/immutable"
"github.com/molecula/featurebase/v2/hash"
"github.com/molecula/featurebase/v2/roaring"
txkey "github.com/molecula/featurebase/v2/short_txkey"
. "github.com/molecula/featurebase/v2/vprint"
@ -1299,27 +1298,6 @@ func (tx *Tx) Min(name string) (uint64, bool, error) {
return uint64((cell.Key << 16) | uint64(cell.firstValue(tx))), true, nil
}
func (tx *Tx) UnionInPlace(name string, others ...*roaring.Bitmap) error {
rbm, err := tx.RoaringBitmap(name)
PanicOn(err)
rbm.UnionInPlace(others...)
// iterate over the containers that changed within rbm, and write them back to disk.
it, found := rbm.Containers.Iterator(0)
_ = found // don't care about the value of found, because first containerKey might be > 0
for it.Next() {
containerKey, rc := it.Value()
// TODO: only write the changed ones back, as optimization?
// Compare to ImportRoaringBits.
err := tx.PutContainer(name, containerKey, rc)
PanicOn(err)
}
return nil
}
// roaring.countRange counts the number of bits set between [start, end).
func (tx *Tx) CountRange(name string, start, end uint64) (uint64, error) {
tx.mu.RLock()
@ -1538,130 +1516,7 @@ func (si *emptyContainerIterator) Value() (uint64, *roaring.Container) {
return 0, nil
}
func (tx *Tx) Dump(short bool, shard uint64) {
fmt.Println(tx.DumpString(short, shard))
}
func (tx *Tx) DumpString(short bool, shard uint64) (r string) {
r = "allkeys:[\n"
// grab root records, for a list of bitmaps.
records, err := tx.RootRecords()
PanicOn(err)
n := 0
for itr := records.Iterator(); !itr.Done(); {
name, _ := itr.Next()
c, err := tx.cursor(name.(string))
PanicOn(err)
defer c.Close()
err = c.First() // First will rewind to beginning.
if err == io.EOF {
r += "<empty bitmap>"
n++
continue
}
PanicOn(err)
for {
err := c.Next()
if err == io.EOF {
break
}
PanicOn(err)
elem := &c.stack.elems[c.stack.top]
leafPage, _, err := c.tx.readPage(elem.pgno)
PanicOn(err)
cell := readLeafCell(leafPage, elem.index)
ckey := cell.Key
ct := toContainer(cell, tx)
s := stringOfCkeyCt(ckey, ct, name.(string), short, true)
r += s
n++
}
}
if n == 0 {
return ""
}
// note that we can have a bitmap present, but it can be empty
r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + "\n"
return "rbf-" + r
}
func containerToBytes(ct *roaring.Container) []byte {
ty := roaring.ContainerType(ct)
switch ty {
case roaring.ContainerNil:
PanicOn("nil container")
case roaring.ContainerArray:
return fromArray16(roaring.AsArray(ct))
case roaring.ContainerBitmap:
return fromArray64(roaring.AsBitmap(ct))
case roaring.ContainerRun:
return fromInterval16(roaring.AsRuns(ct))
}
PanicOn(fmt.Sprintf("unknown container type '%v'", int(ty)))
return nil
}
func bitmapAsString(rbm *roaring.Bitmap) (r string) {
r = "c("
slc := rbm.Slice()
width := 0
s := ""
for _, v := range slc {
if width == 0 {
s = fmt.Sprintf("%v", v)
} else {
s = fmt.Sprintf(", %v", v)
}
width += len(s)
r += s
if width > 70 {
r += ",\n"
width = 0
}
}
if width == 0 && len(r) > 2 {
r = r[:len(r)-2]
}
return r + ")"
}
func stringOfCkeyCt(ckey uint64, ct *roaring.Container, rrName string, short, showHash bool) (s string) {
hsh := ""
if showHash {
by := containerToBytes(ct)
hsh = hash.Blake3sum16(by)
}
cts := roaring.NewSliceContainers()
cts.Put(ckey, ct)
rbm := &roaring.Bitmap{Containers: cts}
srbm := bitmapAsString(rbm)
var pre string
if len(rrName) > 0 {
pre = txkey.PrefixToString([]byte(rrName))
}
bkey := pre + fmt.Sprintf("ckey@%020d", ckey)
s = fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hsh, ct.N())
if !short {
s += " ......." + srbm + "\n"
}
return
}
func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
// begin write boilerplate
if tx.db == nil {

29
rbf/tx_test.go
View file

@ -16,14 +16,12 @@ package rbf_test
import (
"fmt"
"math"
"math/rand"
"sync"
"testing"
"time"
"github.com/molecula/featurebase/v2/rbf"
txkey "github.com/molecula/featurebase/v2/short_txkey"
)
func TestTx_CommitRollback(t *testing.T) {
@ -496,29 +494,6 @@ func BenchmarkTx_Contains(b *testing.B) {
}
}
func TestTx_Dump(t *testing.T) {
db := MustOpenDB(t)
defer MustCloseDB(t, db)
tx := MustBegin(t, db, true)
defer tx.Rollback()
index, field, view, shard := "i", "f", "v", uint64(15)
nm := rbfName(index, field, view, shard)
if err := tx.CreateBitmap(nm); err != nil {
t.Fatal(err)
} else if _, err := tx.Add(nm, 0x00000001, 0x00000002, 0x00010003, 0x00030004); err != nil {
t.Fatal(err)
}
// test that we don't crash, and get *something* back
s := tx.DumpString(true, math.MaxUint64)
if s == "" {
panic("should have had 3 containers!")
}
}
func TestTx_CreateBitmap(t *testing.T) {
t.Run("Bulk", func(t *testing.T) {
db := MustOpenDB(t)
@ -542,7 +517,3 @@ func TestTx_CreateBitmap(t *testing.T) {
}
})
}
func rbfName(index, field, view string, shard uint64) string {
return string(txkey.Prefix(index, field, view, shard))
}

55
rbf/util.go
View file

@ -15,13 +15,16 @@ package rbf
import (
"fmt"
"io"
"strings"
txkey "github.com/molecula/featurebase/v2/short_txkey"
. "github.com/molecula/featurebase/v2/vprint"
)
// we don't currently use dumpAllPages but it's tricky enough to get right
// that it's probably worth keeping as a debugging tool.
var _ = (*Tx).dumpAllPages
func (tx *Tx) dumpAllPages(showLeaves bool) error {
infos, err := tx.PageInfos()
@ -213,56 +216,6 @@ func prefixToString(s string) (ret string) {
return txkey.PrefixToString([]byte(s))
}
func (c *Cursor) dump() {
fmt.Printf("\n Cursor %p has bitmaps:\n%v\n", c, c.debugStringBitmaps())
}
var _ = (&Cursor{}).dump
var _ = (&Cursor{}).debugStringBitmaps
func (c_orig *Cursor) debugStringBitmaps() (r string) {
// work with a totally new Cursor, so we don't impact our current cursor
// so any test using the cursor isn't disturbed.
c2 := Cursor{tx: c_orig.tx}
c2.stack.elems[0] = c_orig.stack.elems[0]
err := c2.First()
if err != nil {
if err == io.EOF {
// ok, can be empty
return "<empty cursor/tx>"
} else {
panic(err)
}
}
n := 0
for {
err := c2.Next()
if err == io.EOF {
break
}
PanicOn(err)
//instead of cell := c2.cell()
elem := &c2.stack.elems[c2.stack.top]
leafPage, _, err := c2.tx.readPage(elem.pgno)
PanicOn(err)
cell := readLeafCell(leafPage, elem.index)
ckey := cell.Key
ct := toContainer(cell, c2.tx)
const short = true
s := stringOfCkeyCt(ckey, ct, "", short, true)
r += s
n++
}
if n == 0 {
return ""
}
return
}
///////////////// happy linter
var _ = printMetaPage

37
roaring/roaring.go
View file

@ -2434,13 +2434,13 @@ func (b *Bitmap) writeOp(op *op) error {
// Iterator returns a new iterator for the bitmap.
func (b *Bitmap) Iterator() *Iterator {
itr := NewIterator(&BitmapIteratorFinder{b})
itr := &Iterator{bitmap: b}
itr.Seek(0)
return itr
}
func (b *Bitmap) IteratorAt(start uint64) *Iterator {
itr := NewIterator(&BitmapIteratorFinder{b})
itr := &Iterator{bitmap: b}
itr.Seek(start)
return itr
}
@ -2701,37 +2701,18 @@ type BitmapInfo struct {
From, To uintptr // if set, indicates the address range used when unpacking
}
type IteratorFinder interface {
FindIterator(uint64) (ContainerIterator, bool)
Close()
}
type BitmapIteratorFinder struct {
bitmap *Bitmap
}
func (bif *BitmapIteratorFinder) FindIterator(seek uint64) (ContainerIterator, bool) {
return bif.bitmap.Containers.Iterator(seek)
}
func (bif *BitmapIteratorFinder) Close() {}
// Iterator represents an iterator over a Bitmap.
type Iterator struct {
finder IteratorFinder
bitmap *Bitmap
citer ContainerIterator
key uint64
c *Container
j, k int32 // i: container; j: array index, bit index, or run index; k: offset within the run
}
// NewIterator requires f as an IteratorFinder, it will
// crash if f is nil.
func NewIterator(f IteratorFinder) *Iterator {
return &Iterator{finder: f}
}
func (itr *Iterator) Close() {
itr.finder.Close()
}
// This exists because we used to support a backend which needed it, and I
// don't want to re-experience the joy of figuring out where close calls are needed.
func (itr *Iterator) Close() {}
// Seek moves to the first value equal to or greater than `seek`.
func (itr *Iterator) Seek(seek uint64) {
@ -2740,7 +2721,7 @@ func (itr *Iterator) Seek(seek uint64) {
itr.k = -1
// Move to the correct container.
itr.citer, _ = itr.finder.FindIterator(highbits(seek))
itr.citer, _ = itr.bitmap.Containers.Iterator(highbits(seek))
if !itr.citer.Next() {
itr.c = nil
return // eof
@ -7537,10 +7518,6 @@ func (c *Container) Difference(other *Container) *Container {
return difference(c, other)
}
func NewSliceContainers() *sliceContainers {
return newSliceContainers()
}
// Slice returns an array of the values in the container as uint16.
// Do NOT modify the result; it could be the container's actual storage.
func (c *Container) Slice() (r []uint16) {

93
rrtx.go
View file

@ -15,9 +15,7 @@
package pilosa
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"sort"
@ -49,27 +47,10 @@ type RoaringTx struct {
w *RoaringWrapper
}
func (tx *RoaringTx) IsDone() (done bool) {
tx.mu.Lock()
done = tx.done
tx.mu.Unlock()
return
}
func (tx *RoaringTx) Type() string {
return RoaringTxn
}
func (tx *RoaringTx) Dump(short bool, shard uint64) {
o := tx.o
o.Shard = shard
fmt.Printf("%v\n", tx.Index.StringifiedRoaringKeys(short, false, o))
}
func (tx *RoaringTx) UseRowCache() bool {
return storage.EnableRowCache()
}
// based on view.openFragments()
func roaringMapOfShards(optionalViewPath string) (shardMap map[uint64]bool, err error) {
@ -112,10 +93,6 @@ func roaringMapOfShards(optionalViewPath string) (shardMap map[uint64]bool, err
return
}
func (tx *RoaringTx) Pointer() string {
return fmt.Sprintf("%p", tx)
}
// NewTxIterator returns a *roaring.Iterator that MUST have Close() called on it BEFORE
// the transaction Commits or Rollsback.
func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
@ -127,33 +104,16 @@ func (tx *RoaringTx) NewTxIterator(index, field, view string, shard uint64) *roa
// ImportRoaringBits return values changed and rowSet will be inaccurate if
// the data []byte is supplied. This mimics the traditional roaring-per-file
// and should be faster.
func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
func (tx *RoaringTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
f, err := tx.getFragment(index, field, view, shard)
if err != nil {
return 0, nil, err
}
if len(data) > 0 {
// changed and rowSet are ignored anyway when len(data) > 0;
// when we are called from fragment.fillFragmentFromArchive()
// which is the only place the data []byte is supplied.
// blueGreenTx also turns off the checks in this case.
return 0, nil, f.readStorageFromArchive(bytes.NewBuffer(data))
}
changed, rowSet, err = f.storage.ImportRoaringRawIterator(rit, clear, true, rowSize)
return
}
func (tx *RoaringTx) Readonly() bool {
return !tx.write
}
func (tx *RoaringTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
frag, err := tx.getFragment(index, field, view, shard)
PanicOn(err)
frag.incrementOpN(changedN)
}
func (c *RoaringTx) ApplyFilter(index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {
return GenericApplyFilter(c, index, field, view, shard, ckey, filter)
}
@ -279,15 +239,6 @@ func (tx *RoaringTx) Min(index, field, view string, shard uint64) (uint64, bool,
return v, ok, nil
}
func (tx *RoaringTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
return err
}
b.UnionInPlace(others...)
return nil
}
func (tx *RoaringTx) CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error) {
b, err := tx.bitmap(index, field, view, shard)
if err != nil {
@ -381,33 +332,6 @@ func (tx *RoaringTx) bitmap(index, field, view string, shard uint64) (*roaring.B
return frag.storage, nil
}
func (tx *RoaringTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
file, err := os.Open(fragmentPathForRoaring) // open the fragment file
if err != nil {
return nil, -1, err
}
fi, err := file.Stat()
if err != nil {
return nil, -1, errors.Wrap(err, "statting")
}
sz = fi.Size()
r = file
return
}
func (tx *RoaringTx) Group() *TxGroup {
return tx.o.Group
}
func (tx *RoaringTx) Options() Txo {
return tx.o
}
// Sn retreives the serial number of the Tx.
func (tx *RoaringTx) Sn() int64 {
return tx.sn
}
func roaringGetFieldView2Shards(idx *Index) (vs *FieldView2Shards, err error) {
vs = NewFieldView2Shards()
@ -651,18 +575,12 @@ func (w *RoaringWrapper) CleanupTx(tx Tx) {
return
}
r.done = true
r.o.dbs.Cleanup(tx) // release the read/write lock.
}
func (w *RoaringWrapper) OpenListString() (r string) {
return "RoaringWrapper.OpenListString() not yet implemented"
}
func (w *RoaringWrapper) OpenSnList() (slc []int64) {
return nil
}
func (w *RoaringWrapper) CloseDB() error {
return errors.New("CloseDB not supported in roaring")
}
@ -721,21 +639,12 @@ func (w *RoaringWrapper) IsClosed() (closed bool) {
return
}
func (w *RoaringWrapper) DeleteDBPath(dbs *DBShard) (err error) {
//vv("RoaringWrapper.DeleteDBPath called on dbs = '%#v'", dbs)
path := dbs.pathForType(roaringTxn)
return os.RemoveAll(path)
}
func (w *RoaringWrapper) DeleteField(index, field, fieldPath string) error {
//vv("RoaringWrapper.DeleteField(index = '%v', field = '%v', fieldPath = '%v'", index, field, fieldPath)
// match txn sn count vs lmdb/etc.
atomic.AddInt64(&globalNextTxSnRoaring, 1)
// under blue-green bolt_roaring, the directory will not be found, b/c bolt will have
// already done the os.RemoveAll(). BUT, RemoveAll returns nil error in this case. Docs:
// "If the path does not exist, RemoveAll returns nil (no error)"
err := os.RemoveAll(fieldPath)
if err != nil {
return errors.Wrap(err, "removing directory")

4
server.go
View file

@ -334,8 +334,8 @@ func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption {
}
// OptServerStorageConfig is a functional option on Server used to specify the
// transactional-storage backend to use, resulting in RoaringTx, RbfTx,
// BadgerTx, or a blueGreen* Tx being used for all Tx interface calls.
// transactional-storage backend to use, resulting in RoaringTx or RbfTx
// being used for all Tx interface calls.
func OptServerStorageConfig(cfg *storage.Config) ServerOption {
return func(s *Server) error {
s.holderConfig.StorageConfig = cfg

24
server/cluster_test.go
View file

@ -142,19 +142,6 @@ func TestClusterResize_EmptyNodes(t *testing.T) {
// Ensure that adding a node correctly resizes the cluster.
func TestClusterResize_AddNode(t *testing.T) {
// Why are we skipping this test under blue-green with Roaring?
//
// We see red test: during resize during importRoaringBits
// PILOSA_STORAGE_BACKEND=rbf_roaring go test -v -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0" -run TestClusterResize_AddNode/"ContinuousShards"
// green:
// PILOSA_STORAGE_BACKEND=roaring_rbf go test -v -tags=' shardwidth20' "-gcflags=all=-d=checkptr=0" -run TestClusterResize_AddNode/"ContinuousShards"
//
// but rbf_badger and badger_rbf are both green (use the same data values for containers).
//
// Conclude: roaring reads a different size of data []byte in (due to ops log) bits vs others (RBF, badger), so
// we can't do blue-green with roaring on this test.
skipTestUnderBlueGreenWithRoaring(t)
t.Run("NoData", func(t *testing.T) {
clus := test.MustRunCluster(t, 3)
defer clus.Close()
@ -344,8 +331,6 @@ func TestClusterResize_AddNode(t *testing.T) {
// Ensure that adding a node correctly resizes the cluster.
func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) {
skipTestUnderBlueGreenWithRoaring(t)
t.Run("WithIndex", func(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
@ -630,12 +615,3 @@ func TestClusterMutualTLS(t *testing.T) {
t.Fatal(err)
}
}
func skipTestUnderBlueGreenWithRoaring(t *testing.T) {
src := pilosa.CurrentBackend()
if strings.Contains(src, "_") {
if strings.Contains(src, "roaring") {
t.Skip("skip for roaring blue-green")
}
}
}

11
server/config.go
View file

@ -207,15 +207,8 @@ type Config struct {
// Storage.Backend determines which Tx implementation the holder/Index will
// use; one of the available transactional-storage engines. Choices are
// listed in the string constants below. Should be one of "roaring","bolt",
// "rbf", "bolt_roaring", "roaring_bolt", "rbf_roaring", "roaring_rbf",
// "bolt_rbf", "rbf_bolt", or any later addition. The engines with _
// underscore indicate use of a blueGreenTx with a comparison of values back
// from each Tx method, and a panic if they differ. This is an effective
// test for consistency. If "rbf_roaring" is specified, then the roaring
// values are the ones actually returned from the blueGreenTx. If
// "roaring_rbf" is chosen, then the RBF values are the ones actually
// returned from the blueGreenTx.
// listed in the string constants below. Should be one of "roaring" or
// "rbf".
Storage *storage.Config `toml:"storage"`
// RowcacheOn, if true, turns on the row cache for all storage backends.

123
stattx.go
View file

@ -16,7 +16,6 @@ package pilosa
import (
"fmt"
"io"
"math"
"runtime"
"sort"
@ -151,8 +150,7 @@ type kall int
// constants for kall argument to callStats.add()
const (
kIncrementOpN kall = iota
kNewTxIterator
kNewTxIterator kall = iota
kImportRoaringBits
kRollback
kCommit
@ -169,23 +167,14 @@ const (
kCount
kMax
kMin
kUnionInPlace
kCountRange
kOffsetRange
kRoaringBitmapReader
kSliceOfShards
kLast // mark the end, always keep this last. The following aren't tracked atm:
kType
kDump
kReadonly
kPointer
kUseRowCache
)
func (k kall) String() string {
switch k {
case kIncrementOpN:
return "kIncrementOpN"
case kNewTxIterator:
return "kNewTxIterator"
case kImportRoaringBits:
@ -220,62 +209,21 @@ func (k kall) String() string {
return "kMax"
case kMin:
return "kMin"
case kUnionInPlace:
return "kUnionInPlace"
case kCountRange:
return "kCountRange"
case kOffsetRange:
return "kOffsetRange"
case kRoaringBitmapReader:
return "kRoaringBitmapReader"
case kSliceOfShards:
return "kSliceOfShards"
case kLast:
return "kLast"
case kType:
return "kType"
case kDump:
return "kDump"
case kReadonly:
return "kReadonly"
case kPointer:
return "kPointer"
case kUseRowCache:
return "kUseRowCache"
}
PanicOn(fmt.Sprintf("unknown kall '%v'", int(k)))
return ""
}
var _ = newStatTx // happy linter
var _ = kPointer
var _ = kUseRowCache
var _ = kType
var _ = kDump
var _ = kReadonly
var _ Tx = (*statTx)(nil)
func (c *statTx) Group() *TxGroup {
return c.b.Group()
}
func (c *statTx) Options() Txo {
return c.b.Options()
}
//IncrementOpN
func (c *statTx) IncrementOpN(index, field, view string, shard uint64, changedN int) {
me := kIncrementOpN
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
c.b.IncrementOpN(index, field, view, shard, changedN)
}
func (c *statTx) NewTxIterator(index, field, view string, shard uint64) *roaring.Iterator {
me := kNewTxIterator
@ -286,7 +234,7 @@ func (c *statTx) NewTxIterator(index, field, view string, shard uint64) *roaring
return c.b.NewTxIterator(index, field, view, shard)
}
func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error) {
func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error) {
me := kImportRoaringBits
t0 := time.Now()
@ -299,25 +247,7 @@ func (c *statTx) ImportRoaringBits(index, field, view string, shard uint64, rit
PanicOn(r)
}
}()
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize, data)
}
func (c *statTx) Dump(short bool, shard uint64) {
c.b.Dump(short, shard)
}
func (c *statTx) Readonly() bool {
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see Readonly() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.Readonly()
}
func (tx *statTx) Pointer() string {
return fmt.Sprintf("%p", tx)
return c.b.ImportRoaringBits(index, field, view, shard, rit, clear, log, rowSize)
}
func (c *statTx) Rollback() {
@ -421,14 +351,6 @@ func (c *statTx) RemoveContainer(index, field, view string, shard uint64, key ui
return c.b.RemoveContainer(index, field, view, shard, key)
}
func (c *statTx) UseRowCache() bool {
return c.b.UseRowCache()
}
func (c *statTx) IsDone() (done bool) {
return c.b.IsDone()
}
func (c *statTx) Add(index, field, view string, shard uint64, a ...uint64) (changeCount int, err error) {
me := kAdd
@ -586,23 +508,6 @@ func (c *statTx) Min(index, field, view string, shard uint64) (uint64, bool, err
return c.b.Min(index, field, view, shard)
}
func (c *statTx) UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error {
me := kUnionInPlace
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see UnionInPlace() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.UnionInPlace(index, field, view, shard, others...)
}
func (c *statTx) CountRange(index, field, view string, shard uint64, start, end uint64) (n uint64, err error) {
me := kCountRange
@ -636,32 +541,10 @@ func (c *statTx) OffsetRange(index, field, view string, shard, offset, start, en
return c.b.OffsetRange(index, field, view, shard, offset, start, end)
}
func (c *statTx) RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error) {
me := kRoaringBitmapReader
t0 := time.Now()
defer func() {
c.stats.add(me, time.Since(t0))
}()
defer func() {
if r := recover(); r != nil {
AlwaysPrintf("see RoaringBitmapReader() PanicOn '%v' at '%v'", r, Stack())
PanicOn(r)
}
}()
return c.b.RoaringBitmapReader(index, field, view, shard, fragmentPathForRoaring)
}
func (c *statTx) Type() string {
return c.b.Type()
}
// Sn retreives the serial number of the Tx.
func (c *statTx) Sn() int64 {
return c.b.Sn()
}
func (c *statTx) GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error) {
return c.b.GetSortedFieldViewList(idx, shard)
}

2
storage/cache.go
View file

@ -31,6 +31,6 @@ func SetRowCacheOn(on bool) {
}
}
func EnableRowCache() bool {
func RowCacheEnabled() bool {
return atomic.LoadInt64(&enableRowcache) == 1
}

107
tx.go
View file

@ -15,8 +15,6 @@
package pilosa
import (
"io"
"github.com/molecula/featurebase/v2/roaring"
txkey "github.com/molecula/featurebase/v2/short_txkey"
//txkey "github.com/molecula/featurebase/v2/txkey"
@ -45,8 +43,8 @@ const writable = true
// that have not been committed.
type Tx interface {
// Type returns "roaring", "rbf", "bolt", "badger_roaring", or one of the other
// blue-green Tx types at the top of txfactory.go
// Type returns "roaring", "rbf", or one of the other
// Tx types at the top of txfactory.go
Type() string
// Rollback must be called the end of read-only transactions. Either
@ -65,32 +63,6 @@ type Tx interface {
// Commit makes the updates in the Tx visible to subsequent transactions.
Commit() error
// IsDone must return true if Rollback() or Commit() has already
// been called. Otherwise it must return false. This allows
// DBWrapper.CleanupTx(tx Tx) to be idempotent.
IsDone() bool
// Readonly returns the flag this transaction was created with
// during NewTx. If the transaction is writable, it will return false.
Readonly() bool
// UseRowCache is used by fragment.go unprotectedRow() to determine
// dynamically at runtime if RoaringTx
// are in use, which for continuity wants to continue to use the
// rowCache, or if other storage engines (RBF, Badger) are in
// use, which will mean that the bitmap data stored by the
// rowCache can disappear as it is un-mmap-ed, causing crashes.
UseRowCache() bool
// IncrementOpN updates internal statistics with the changedN provided.
IncrementOpN(index, field, view string, shard uint64, changedN int)
// Pointer gives us a memory address for the underlying
// transaction for debugging.
// It is public because we use it in roaring to report invalid
// container memory access outside of a transaction.
Pointer() string
// NewTxIterator returns it, a *roaring.Iterator whose it.Next() will
// successively return each uint64 stored in the conceptual roaring.Bitmap
// for the specified fragment.
@ -103,8 +75,7 @@ type Tx interface {
// Return value 'found' is true when the ckey container was present.
// ckey of 0 gives all containers (in the fragment).
//
// ContainerIterator must not have side-effects. blueGreenTx will
// call it at the very beginning of commit to verify db contents.
// ContainerIterator must not have side-effects.
//
// citer.Close() must be called when the client is done using it.
ContainerIterator(index, field, view string, shard uint64, ckey uint64) (citer roaring.ContainerIterator, found bool, err error)
@ -154,9 +125,6 @@ type Tx interface {
// Min
Min(index, field, view string, shard uint64) (uint64, bool, error)
// UnionInPlace
UnionInPlace(index, field, view string, shard uint64, others ...*roaring.Bitmap) error
// CountRange
CountRange(index, field, view string, shard uint64, start, end uint64) (uint64, error)
@ -170,31 +138,9 @@ type Tx interface {
// If clear is true, the bits from rit are cleared, otherwise they are set in the
// specifed fragment.
//
// The data argument can be nil, its ignored for RBF/BadgerTx. It is supplied to
// RoaringTx.ImportRoaringBits() in fragment.go fragment.fillFragmentFromArchive()
// to do the traditional fragment.readStorageFromArchive() which
// does some in memory field/view/fragment metadata updates.
// It makes blueGreenTx testing viable too.
//
// ImportRoaringBits return values changed and rowSet may be inaccurate if
// the data []byte is supplied (the RoaringTx implementation neglects this for speed).
ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64, data []byte) (changed int, rowSet map[uint64]int, err error)
RoaringBitmapReader(index, field, view string, shard uint64, fragmentPathForRoaring string) (r io.ReadCloser, sz int64, err error)
// Group returns nil or the TxGroup that this Tx is a part of.
Group() *TxGroup
// Dump is for debugging, what does this Tx see as its database?
Dump(short bool, shard uint64)
// Options returns the options used to create this Tx. This
// can be implementd by embedding Txo, and Txo provides the
// Options() method.
Options() Txo
// Sn retreives the serial number of the Tx.
Sn() int64
ImportRoaringBits(index, field, view string, shard uint64, rit roaring.RoaringIterator, clear bool, log bool, rowSize uint64) (changed int, rowSet map[uint64]int, err error)
// GetSortedFieldViewList gets the set of FieldView(s)
GetSortedFieldViewList(idx *Index, shard uint64) (fvs []txkey.FieldView, err error)
@ -202,51 +148,6 @@ type Tx interface {
GetFieldSizeBytes(index, field string) (uint64, error)
}
// Closer is used by Finders
type Closer interface {
Close()
}
type Dumper interface {
// Dump is for debugging, what does this Tx see as its database?
AllDump()
}
// TxStore has operations that will create and commit multiple
// Tx on a backing store.
type TxStore interface {
// DeleteFragment deletes all the containers in a fragment.
//
// This is not in a Tx because it will often do too many deletes for a single
// transaction, and clients would be suprised to find their Tx had already
// been commited and they are getting an error on double-Commit.
// Instead each TxStore implementation creates and commits as many
// transactions as needed.
//
// Argument frag should be passed by any RoaringTx user, but for RBF/Badger it can be nil.
// If not nil, it must be of type *fragment. If frag is supplied, then
// index must be equal to frag.index, field equal to frag.field, view equal
// to frag.view, and shard equal to frag.shard.
//
DeleteFragment(index, field, view string, shard uint64, frag interface{}) error
DeleteField(index, field string) error
// Close shuts down the database.
Close() error
}
// RawRoaringData used by ImportRoaringBits.
// must be consumable by roaring.newRoaringIterator()
type RawRoaringData struct {
data []byte
}
func (rr *RawRoaringData) Iterator() (roaring.RoaringIterator, error) {
return roaring.NewRoaringIterator(rr.data)
}
// GenericApplyFilter implements ApplyFilter in terms of tx.ContainerIterator,
// as a convenience if a Tx backend hasn't implemented this new function yet.
func GenericApplyFilter(tx Tx, index, field, view string, shard uint64, ckey uint64, filter roaring.BitmapFilter) (err error) {

689
txfactory.go
View file

@ -16,31 +16,22 @@ package pilosa
import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
"syscall"
"text/tabwriter"
"github.com/molecula/featurebase/v2/hash"
"github.com/molecula/featurebase/v2/roaring"
txkey "github.com/molecula/featurebase/v2/short_txkey"
"github.com/molecula/featurebase/v2/storage"
"github.com/molecula/featurebase/v2/testhook"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
"github.com/zeebo/blake3"
)
// public strings that pilosa/server/config.go can reference
const (
RoaringTxn string = "roaring"
RBFTxn string = "rbf"
BoltTxn string = "bolt"
)
// DetectMemAccessPastTx true helps us catch places in api and executor
@ -147,6 +138,11 @@ func (q *Qcx) Finish() (err error) {
}
}
err2 := q.Grp.FinishGroup()
// drop the old group so we aren't holding references to all those Tx
q.Grp = q.Txf.NewTxGroup()
if !q.done {
_ = testhook.Closed(q.Txf.holder.Auditor, q, nil)
}
q.done = true
if err != nil {
@ -164,7 +160,11 @@ func (q *Qcx) Abort() {
(*q.RequiredForAtomicWriteTx).Rollback()
}
q.Grp.AbortGroup()
// drop the old group so we aren't holding references to all those Tx
q.Grp = q.Txf.NewTxGroup()
if !q.done {
_ = testhook.Closed(q.Txf.holder.Auditor, q, nil)
}
q.done = true
}
@ -197,6 +197,7 @@ func (f *TxFactory) NewQcx() (qcx *Qcx) {
if f.typeOfTx == "roaring" {
qcx.isRoaring = true
}
_ = testhook.Opened(f.holder.Auditor, qcx, nil)
return
}
@ -256,7 +257,7 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) {
// qcx.write reflects the top executor determination
// if a write will be done at the end, so we upgrade
// the "local" read Tx to be writes, so that they
// don't deadlock against themselves under blue-green.
// don't deadlock against themselves.
o.Write = o.Write || qcx.write
// In general, we make ALL write transactions local, and never reuse them
@ -294,9 +295,8 @@ func (qcx *Qcx) GetTx(o Txo) (tx Tx, finisher func(perr *error), err error) {
if already {
return
}
o.Group = qcx.Grp
tx = qcx.Txf.NewTx(o)
qcx.Grp.AddTx(tx)
qcx.Grp.AddTx(tx, o)
return
}
@ -340,7 +340,6 @@ func (qcx *Qcx) StartAtomicWriteTx(o Txo) {
// new Tx needed
tx := qcx.Txf.NewTx(o)
qcx.RequiredForAtomicWriteTx = &tx
o := tx.Options()
qcx.RequiredTxo = &o
return
}
@ -363,55 +362,23 @@ func (qcx *Qcx) StartAtomicWriteTx(o Txo) {
}
}
func (qcx *Qcx) SetRequiredForAtomicWriteTx(tx Tx) {
if tx == nil || NilInside(tx) {
PanicOn("cannot set nil tx in SetRequiredForAtomicWriteTx")
}
qcx.mu.Lock()
qcx.RequiredForAtomicWriteTx = &tx
o := tx.Options()
qcx.RequiredTxo = &o
qcx.mu.Unlock()
}
func (qcx *Qcx) ClearRequiredForAtomicWriteTx() {
qcx.mu.Lock()
qcx.RequiredForAtomicWriteTx = nil
qcx.RequiredTxo = nil
qcx.mu.Unlock()
}
func (qcx *Qcx) ListOpenTx() string {
return qcx.Grp.String()
}
// TxFactory abstracts the creation of Tx interface-level
// transactions so that RBF, BoltDB, or Roaring-fragment-files, or several
// transactions so that RBF, or Roaring-fragment-files, or several
// of these at once in parallel, is used as the storage and transction layer.
type TxFactory struct {
typeOfTx string
mu sync.Mutex
types []txtype // blue-green split individually here
typ txtype
dbsClosed bool // idemopotent CloseDB()
dbPerShard *DBPerShard
holder *Holder
blueGreenReg *blueGreenRegistry
// allow holder to activate blue-green checking only
// once we have synced both sides at start up time.
blueGreenOff bool
isBlueGreen bool
}
func (f *TxFactory) Types() []txtype {
return f.types
}
// integer types for fast switch{}
@ -421,7 +388,6 @@ const (
noneTxn txtype = 0
roaringTxn txtype = 1 // these don't really have any transactions
rbfTxn txtype = 2
boltTxn txtype = 4
)
// DirectoryName just returns a string version of the transaction type. We
@ -434,73 +400,41 @@ func (ty txtype) DirectoryName() string {
return "roaring"
case rbfTxn:
return "rbf"
case boltTxn:
return "boltdb"
}
PanicOn(fmt.Sprintf("unkown txtype %v", int(ty)))
return ""
}
func (txf *TxFactory) NeedsSnapshot() (b bool) {
for _, ty := range txf.types {
switch ty {
case roaringTxn:
b = true
return
}
}
return
return txf.typ == roaringTxn
}
func MustBackendToTxtype(backend string) (types []txtype) {
var srcs []string
func MustBackendToTxtype(backend string) (typ txtype) {
if strings.Contains(backend, "_") {
srcs = strings.Split(backend, "_")
if len(srcs) != 2 {
PanicOn("only two blue-green comparisons permitted")
}
} else {
srcs = append(srcs, backend)
panic("blue-green comparisons removed")
}
for i, s := range srcs {
switch s {
case RoaringTxn: // "roaring"
types = append(types, roaringTxn)
case RBFTxn: // "rbf"
types = append(types, rbfTxn)
case BoltTxn: // "bolt"
types = append(types, boltTxn)
default:
PanicOn(fmt.Sprintf("unknown backend '%v'", s))
}
if i == 1 {
if types[1] == types[0] {
PanicOn(fmt.Sprintf("cannot blue-green the same backend on both arms: '%v'", s))
}
}
switch backend {
case RoaringTxn: // "roaring"
return roaringTxn
case RBFTxn: // "rbf"
return rbfTxn
}
return
panic(fmt.Sprintf("unknown backend '%v'", backend))
}
// NewTxFactory always opens an existing database. If you
// want to a fresh database, os.RemoveAll on dir/name ahead of time.
// We always store files in a subdir of holderDir.
func NewTxFactory(backend string, holderDir string, holder *Holder) (f *TxFactory, err error) {
types := MustBackendToTxtype(backend)
typ := MustBackendToTxtype(backend)
f = &TxFactory{
types: types,
typ: typ,
typeOfTx: backend,
holder: holder,
}
if len(types) == 2 {
f.blueGreenReg = newBlueGreenReg(types)
f.isBlueGreen = true
// blue-green can never use the rowCache.
storage.SetRowCacheOn(false)
}
f.dbPerShard = f.NewDBPerShard(types, holderDir, holder)
f.dbPerShard = f.NewDBPerShard(typ, holderDir, holder)
if f.hasRBF() {
holder.Logger.Infof("rbf config = %#v", holder.cfg.RBFConfig)
@ -515,15 +449,6 @@ func (f *TxFactory) Open() error {
return f.dbPerShard.LoadExistingDBs()
}
// UseRowCache can be more "global" than Tx at the moment, because
// we are sharing the same bool flag in rbf at the moment. If
// this changes then fragment.openStorage() will need a new way
// to determine if it should use the rowCache. Currently it
// doesn't have a tx Tx parameter, so we use the Txf instead.
func (f *TxFactory) UseRowCache() bool {
return storage.EnableRowCache()
}
// Txo holds the transaction options
type Txo struct {
Write bool
@ -533,23 +458,14 @@ type Txo struct {
Shard uint64
dbs *DBShard
per *DBPerShard
Group *TxGroup
blueGreenOff bool
}
func (o Txo) String() string {
return fmt.Sprintf("Txo{Write:%v, Index:%v Shard:%v Group:%p}", o.Write, o.Index.name, o.Shard, o.Group)
}
func (f *TxFactory) TxType() string {
return f.typeOfTx
}
func (f *TxFactory) TxTypes() []txtype {
return f.types
func (f *TxFactory) TxTyp() txtype {
return f.typ
}
func (f *TxFactory) DeleteIndex(name string) (err error) {
@ -566,10 +482,6 @@ func (f *TxFactory) DeleteFragmentFromStore(
return f.dbPerShard.DeleteFragment(index, field, view, shard, frag)
}
func (f *TxFactory) DumpAll() {
f.dbPerShard.DumpAll()
}
// IndexUsageDetails computes the sum of filesizes used by the node, broken down
// by index, field, fragments and keys.
func (f *TxFactory) IndexUsageDetails(isClosing func() bool) (map[string]IndexUsage, uint64, error) {
@ -758,7 +670,6 @@ func directoryUsage(fname string, recursive bool) (uint64, error) {
// CloseIndex is a no-op. This seems to be in place for debugging purposes.
func (f *TxFactory) CloseIndex(idx *Index) error {
//idx.Dump("CloseIndex")
return nil
}
@ -779,24 +690,24 @@ func init() {
}
}
// TxGroup holds a set of read and a set of write transactions
// that will en-mass have Rollback() (for the read set) and
// Commit() (for the write set) called on
// TxGroup holds a set of read transactions
// that will en-mass have Rollback() (for the read set) called on
// them when TxGroup.Finish() is invoked.
// Alternatively, TxGroup.Abort() will call Rollback()
// on all Tx group memebers.
//
// It used to have writes but we never actually used that because
// of the Qcx needing to make every commit get its own transaction.
type TxGroup struct {
mu sync.Mutex
fac *TxFactory
reads []Tx
writes []Tx
finished bool
all map[grpkey]Tx
}
type grpkey struct {
write bool
index string
shard uint64
}
@ -811,7 +722,7 @@ func (g *TxGroup) AlreadyHaveTx(o Txo) (tx Tx, already bool) {
mustHaveIndexShard(&o)
g.mu.Lock()
defer g.mu.Unlock()
key := grpkey{write: o.Write, index: o.Index.name, shard: o.Shard}
key := grpkey{index: o.Index.name, shard: o.Shard}
tx, already = g.all[key]
return
}
@ -819,21 +730,14 @@ func (g *TxGroup) AlreadyHaveTx(o Txo) (tx Tx, already bool) {
func (g *TxGroup) String() (r string) {
g.mu.Lock()
defer g.mu.Unlock()
if len(g.reads) == 0 && len(g.writes) == 0 {
if len(g.reads) == 0 {
return "<empty-TxGroup>"
}
i := 0
r += "\n"
for _, tx := range g.reads {
r += fmt.Sprintf("[%v]read: _sn_ %v %v, \n", i, tx.Sn(), tx.Options())
i++
for i, tx := range g.reads {
r += fmt.Sprintf("[%v]read: %#v,\n", i, tx)
}
for _, tx := range g.writes {
r += fmt.Sprintf("[%v]write: _sn_ %v %v, \n", i, tx.Sn(), tx.Options())
i++
}
return
return r
}
// NewTxGroup
@ -846,7 +750,7 @@ func (f *TxFactory) NewTxGroup() (g *TxGroup) {
}
// AddTx adds tx to the group.
func (g *TxGroup) AddTx(tx Tx) {
func (g *TxGroup) AddTx(tx Tx, o Txo) {
g.mu.Lock()
defer g.mu.Unlock()
if g.finished {
@ -856,15 +760,9 @@ func (g *TxGroup) AddTx(tx Tx) {
PanicOn("Cannot add nil Tx to TxGroup")
}
if tx.Readonly() {
g.reads = append(g.reads, tx)
} else {
g.writes = append(g.writes, tx)
}
o := tx.Options()
mustHaveIndexShard(&o)
g.reads = append(g.reads, tx)
key := grpkey{write: o.Write, index: o.Index.name, shard: o.Shard}
key := grpkey{index: o.Index.name, shard: o.Shard}
prior, ok := g.all[key]
if ok {
PanicOn(fmt.Sprintf("already have Tx in group for this, we should have re-used it! prior is '%v'; tx='%v'", prior, tx))
@ -882,15 +780,6 @@ func (g *TxGroup) FinishGroup() (err error) {
PanicOn("in TxGroup.Finish(): TxGroup already finished")
}
g.finished = true
for i, tx := range g.writes {
_ = i
err0 := tx.Commit()
if err0 != nil {
if err == nil {
err = err0 // keep the first error, but Commit them all.
}
}
}
for _, r := range g.reads {
r.Rollback()
}
@ -912,9 +801,6 @@ func (g *TxGroup) AbortGroup() {
for _, r := range g.reads {
r.Rollback()
}
for _, tx := range g.writes {
tx.Rollback()
}
}
func (f *TxFactory) NewTx(o Txo) (txn Tx) {
@ -924,12 +810,6 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) {
}
}()
if f.isBlueGreen {
f.mu.Lock()
o.blueGreenOff = f.blueGreenOff
f.mu.Unlock()
}
indexName := ""
if o.Index != nil {
indexName = o.Index.name
@ -952,10 +832,8 @@ func (f *TxFactory) NewTx(o Txo) (txn Tx) {
if dbs.Shard != o.Shard {
PanicOn(fmt.Sprintf("asked for o.Shard=%v but got dbs.Shard=%v", int(o.Shard), int(dbs.Shard)))
}
//vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.types='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.types, dbs.W)
o.dbs = dbs // our specific database per shard.
o.per = f.dbPerShard // for top level debug Dumps
//vv("got dbs='%p' for o.Index='%v'; shard='%v'; dbs.typ='%#v'; dbs.W='%#v'", dbs, o.Index.name, o.Shard, dbs.typ, dbs.W)
o.dbs = dbs
tx, err := dbs.NewTx(o.Write, indexName, o)
if err != nil {
@ -973,8 +851,6 @@ func (ty txtype) String() string {
return "roaring"
case rbfTxn:
return "rbf"
case boltTxn:
return "bolt"
}
PanicOn(fmt.Sprintf("unhandled ty '%v' in txtype.String()", int(ty)))
return ""
@ -1011,147 +887,6 @@ func fragmentSpecFromRoaringPath(path string) (field, view string, shard uint64,
return
}
// hashOnly means only show the value hash, not the content bits.
// showOps means display the ops log.
func (idx *Index) StringifiedRoaringKeys(hashOnly, showOps bool, o Txo) (r string) {
paths, err := listFilesUnderDir(idx.path, false, "", true)
PanicOn(err)
index := idx.name
r = "allkeys:[\n"
n := 0
for _, relpath := range paths {
field, view, shard, err := fragmentSpecFromRoaringPath(relpath)
if err != nil {
continue // ignore .meta paths
}
if shard != o.Shard {
continue // only print the shard the Txo is on.
}
abspath := idx.path + sep + relpath
s, _, err := stringifiedRawRoaringFragment(abspath, index, field, view, shard, showOps, hashOnly, os.Stdout)
PanicOn(err)
//r += fmt.Sprintf("path:'%v' fragment contains:\n") + s
//if s == "" {
//s = "<empty bitmap>"
//}
r += s
n++
}
if n == 0 {
return "<empty roaring data>"
}
// note that we can have a bitmap present, but it can be empty
r += "]\n all-in-blake3:" + hash.Blake3sum16([]byte(r)) + "\n"
return "roaring-" + r
}
func RoaringFragmentChecksum(path string, index, field, view string, shard uint64) (r string, hotbits int) {
defer func() {
r := recover()
if r != nil {
PanicOn(fmt.Sprintf("caught PanicOn on path='%v', index='%v', field='%v', view='%v', shard='%v': %v",
path, index, field, view, shard, r))
}
}()
hasher := blake3.New()
showOps := false
hashOnly := true
hash, hotbits, err := stringifiedRawRoaringFragment(path, index, field, view, shard, showOps, hashOnly, hasher)
PanicOn(err)
fmt.Fprintf(hasher, "%v/%v/%v/%v/%v", index, field, view, shard, hash)
var buf [16]byte
_, _ = hasher.Digest().Read(buf[0:])
return fmt.Sprintf("%x", buf), hotbits
}
func stringifiedRawRoaringFragment(path string, index, field, view string, shard uint64, showOps, hashOnly bool, w io.Writer) (r string, hotbits int, err error) {
var info roaring.BitmapInfo
_ = info
var f *os.File
f, err = os.Open(path)
PanicOn(err)
if err != nil {
return
}
var fi os.FileInfo
fi, err = f.Stat()
PanicOn(err)
if err != nil {
return
}
// Memory map the file.
data, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
if err != nil {
err = errors.Wrap(err, "mmapping")
return
}
defer func() {
err := syscall.Munmap(data)
if err != nil {
PanicOn(fmt.Errorf("loadRawRoaringContainer: munmap failed: %v", err))
}
PanicOn(f.Close())
}()
// Attach the mmap file to the bitmap.
var rbm *roaring.Bitmap
rbm, _, err = roaring.InspectBinary(data, true, &info)
if err != nil {
err = errors.Wrap(err, "inspecting")
return
}
//cmd.DisplayInfo(info)
// inlined
if showOps {
pC := pointerContext{
from: info.From,
to: info.To,
}
if info.ContainerCount > 0 {
printContainers(w, info, pC)
}
if info.Ops > 0 {
printOps(w, info)
}
}
citer, found := rbm.Containers.Iterator(0)
_ = found // probably gonna use just the Ops log instead, so don't PanicOn if !found.
for citer.Next() {
ckey, ct := citer.Value()
by := containerToBytes(ct)
hash := hash.Blake3sum16(by)
cts := roaring.NewSliceContainers()
cts.Put(ckey, ct)
rbm := &roaring.Bitmap{Containers: cts}
var srbm string
if !hashOnly {
srbm = BitmapAsString(rbm)
}
bkey := txkey.ToString(txkey.Key(index, field, view, shard, ckey))
n := ct.N()
hotbits += int(n)
r += fmt.Sprintf("%v -> %v (%v hot)\n", bkey, hash, n)
if !hashOnly {
r += " ......." + srbm + "\n"
}
}
return
}
// listFilesUnderDir returns the paths of files found under directory root.
// If includeRoot is true, it returns the full path, otherwise paths are relative to root.
// If requriedSuffix is supplied, the returned file paths will end in that,
@ -1207,130 +942,6 @@ func fileSize(name string) (int64, error) {
return fi.Size(), nil
}
var _ = fileSize // happy linter
func containerToBytes(ct *roaring.Container) []byte {
ty := roaring.ContainerType(ct)
switch ty {
case roaring.ContainerNil:
PanicOn("nil roaring.Container")
case roaring.ContainerArray:
return fromArray16(roaring.AsArray(ct))
case roaring.ContainerBitmap:
return fromArray64(roaring.AsBitmap(ct))
case roaring.ContainerRun:
return fromInterval16(roaring.AsRuns(ct))
}
PanicOn(fmt.Sprintf("unknown roaring.Container type '%v'", int(ty)))
return nil
}
type pointerContext struct {
from, to uintptr
}
func printOps(w io.Writer, info roaring.BitmapInfo) {
fmt.Fprintln(w, " Ops:")
tw := tabwriter.NewWriter(w, 0, 8, 0, '\t', 0)
fmt.Fprintf(tw, " \t%s\t%s\t%s\t\n", "TYPE", "OpN", "SIZE")
printed := 0
for _, op := range info.OpDetails {
fmt.Fprintf(tw, "\t%s\t%d\t%d\t\n", op.Type, op.OpN, op.Size)
printed++
}
tw.Flush()
}
func (p *pointerContext) pretty(c roaring.ContainerInfo) string {
var pointer string
if c.Mapped {
if c.Pointer >= p.from && c.Pointer < p.to {
pointer = fmt.Sprintf("@+0x%x", c.Pointer-p.from)
} else {
pointer = fmt.Sprintf("!0x%x!", c.Pointer)
}
} else {
pointer = fmt.Sprintf("0x%x", c.Pointer)
}
return fmt.Sprintf("%s \t%d \t%d \t%s ", c.Type, c.N, c.Alloc, pointer)
}
// stolen from ctl/inspect.go
func printContainers(w io.Writer, info roaring.BitmapInfo, pC pointerContext) {
fmt.Fprintln(w, " Containers:")
tw := tabwriter.NewWriter(w, 0, 8, 0, '\t', 0)
fmt.Fprintf(tw, " \t\tRoaring\t\t\t\tOps\t\t\t\tFlags\t\n")
fmt.Fprintf(tw, "\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t\n", "KEY", "TYPE", "N", "ALLOC", "OFFSET", "TYPE", "N", "ALLOC", "OFFSET", "FLAGS")
c1s := info.Containers
c2s := info.OpContainers
l1 := len(c1s)
l2 := len(c2s)
i1 := 0
i2 := 0
var c1, c2 roaring.ContainerInfo
c1.Key = ^uint64(0)
c2.Key = ^uint64(0)
c1e := false
c2e := false
if i1 < l1 {
c1 = c1s[i1]
i1++
c1e = true
}
if i2 < l2 {
c2 = c2s[i2]
i2++
c2e = true
}
printed := 0
for c1e || c2e {
c1used := false
c2used := false
var key uint64
c1fmt := "-\t\t\t"
c2fmt := "-\t\t\t"
// If c2 exists, we'll always prefer its flags,
// if it doesn't, this gets overwritten.
flags := c2.Flags
if !c2e || (c1e && c1.Key < c2.Key) {
c1fmt = pC.pretty(c1)
key = c1.Key
c1used = true
flags = c1.Flags
} else if !c1e || (c2e && c2.Key < c1.Key) {
c2fmt = pC.pretty(c2)
key = c2.Key
c2used = true
} else {
// c1e and c2e both set, and neither key is < the other.
c1fmt = pC.pretty(c1)
c2fmt = pC.pretty(c2)
key = c1.Key
c1used = true
c2used = true
}
if c1used {
if i1 < l1 {
c1 = c1s[i1]
i1++
} else {
c1e = false
}
}
if c2used {
if i2 < l2 {
c2 = c2s[i2]
i2++
} else {
c2e = false
}
}
fmt.Fprintf(tw, "\t%d\t%s\t%s\t%s\t\n", key, c1fmt, c2fmt, flags)
printed++
}
tw.Flush()
}
var _ = anyGlobalDBWrappersStillOpen // happy linter
func anyGlobalDBWrappersStillOpen() bool {
@ -1340,225 +951,19 @@ func anyGlobalDBWrappersStillOpen() bool {
if globalRbfDBReg.Size() != 0 {
return true
}
if globalBoltReg.Size() != 0 {
return true
}
return false
}
func (f *TxFactory) blueGreenOnIfRunningBlueGreen() {
if len(f.types) == 2 {
f.blueGreenOff = false
}
}
func (f *TxFactory) blueGreenOffIfRunningBlueGreen() {
if len(f.types) == 2 {
f.blueGreenOff = true
}
}
func (f *TxFactory) hasRoaring() bool {
return f.types[0] == roaringTxn || (len(f.types) > 1 && f.types[1] == roaringTxn)
return f.typ == roaringTxn
}
func (f *TxFactory) hasRBF() bool {
return f.types[0] == rbfTxn || (len(f.types) > 1 && f.types[1] == rbfTxn)
return f.typ == rbfTxn
}
var _ = (&TxFactory{}).hasRoaring // happy linter
func (f *TxFactory) blueHasData() (hasData bool, err error) {
if len(f.types) != 2 {
return false, nil
}
return f.dbPerShard.HasData(0)
}
func (f *TxFactory) greenHasData() (hasData bool, err error) {
n := len(f.types)
switch n {
case 1:
return f.dbPerShard.HasData(0)
case 2:
return f.dbPerShard.HasData(1)
}
err = fmt.Errorf("unsupported len(f.types): %v; must be 1 or 2", n)
PanicOn(err)
return
}
// green2blue is called at the very end of Holder.Open(), so
// we know that the holder is ready to go, knowing its holder.Indexes(), fields,
// view, shards, and other metadata if any.
//
// Called by test Test_TxFactory_UpdateBlueFromGreen_OnStartup() in
// txfactory_internal_test.go as well.
//
// This is a noop if we aren't running under a blue_green PILOSA_STORAGE_BACKEND.
func (f *TxFactory) green2blue(holder *Holder) (err0 error) {
// Holder.Open will always call us, even without blue_green. Which is fine.
// We are just a no-op in that case.
if len(f.types) != 2 {
return nil
}
holder.Logger.Infof("green2blue analysis begins.")
blueDest := f.types[0]
greenSrc := f.types[1]
if blueDest == roaringTxn {
return fmt.Errorf("error: cannot migrate to 'roaring': not implemented")
}
idxs := holder.Indexes()
verifyInsteadOfCopy := false
blueHasData, err := f.blueHasData()
if err != nil {
return errors.Wrap(err, "TxFactory.green2blue f.blueHasData()")
}
greenHasData, err := f.greenHasData()
if err != nil {
return errors.Wrap(err, "TxFactory.green2blue f.greenHasData()")
}
if !blueHasData && !greenHasData {
holder.Logger.Infof("no data in blue or green. No migration or verification to do")
return nil
}
// INVAR: blue has data.
if !greenHasData {
holder.Logger.Errorf("cannot migrate from green '%v' because it has no data in it", greenSrc)
return fmt.Errorf("error: cannot migrate from green '%v' because it has no data in it", greenSrc)
}
nGoro := runtime.NumCPU()
if nGoro < 5 {
// try to get some overlapped IO
nGoro = 5
}
pj := newParallelJobs(nGoro)
action := "verify"
if blueHasData {
verifyInsteadOfCopy = true
defer holder.Logger.Infof("bitmap-backend verification done : %v compared to %v", blueDest, greenSrc)
} else {
action = "migrate"
holder.Logger.Infof("bitmap-backend migration starting: populating %v from %v with %v threads", blueDest, greenSrc, nGoro)
defer holder.Logger.Infof("bitmap-backend migration done : populated %v from %v", blueDest, greenSrc)
}
firstPjobStarted := false
indexloop:
for k, idx := range idxs {
// scan directories
blueShards, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(blueDest, idx, "", false)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v') error fetching blueShards", idx.name))
}
// scan directories
greenShards, err := f.dbPerShard.TypedDBPerShardGetShardsForIndex(greenSrc, idx, "", true)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v') error fetching greenShards", idx.name))
}
if verifyInsteadOfCopy {
diff := f.shardSetDiff(blueShards, greenShards)
if diff != "" {
return fmt.Errorf("verifyInsteadOfCopy true, blue[%v]=%#v and green[%v]=%#v have different shards for index '%v': '%v'; stack=\n%v", blueDest, blueShards, greenSrc, greenShards, idx.name, diff, Stack())
}
// can also check against meta data
shards := idx.AvailableShards(localOnly).Slice()
meta := make(map[uint64]bool)
for _, shard := range shards {
meta[shard] = true
}
diff2 := f.shardSetDiff(greenShards, meta)
if diff2 != "" {
return fmt.Errorf("green[%v] = '%#v' and meta data '%#v' have different shards for index '%v': %v", greenSrc, greenShards, shards, idx.name, diff2)
}
}
shardNum := 0
for shard := range greenShards {
shardNum++
shnum := shardNum
idx := idx
shard := shard
k := k
fun := func(worker int) error {
dbs, err := f.dbPerShard.GetDBShard(idx.name, shard, idx)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("GetDBShard(index='%v', shard='%v')", idx.name, int(shard)))
}
holder.Logger.Infof("%v progress on index '%v' (%v of %v): on shard '%v' (%v of %v) [worker %v]",
action, idx.name, k+1, len(idxs), shard, shnum, len(greenShards), worker)
if verifyInsteadOfCopy {
// verify all containers
err = dbs.verifyBlueEqualsGreen()
if err != nil {
return errors.Wrap(err,
fmt.Sprintf("dbs.verifyBlueEqualsGreen(blue='%v', "+
"green='%v') for index='%v', shard='%v'",
blueDest, greenSrc, idx.name, int(shard)))
}
} else {
// the main copy work
err = dbs.populateBlueFromGreen()
if err != nil {
return errors.Wrap(err,
fmt.Sprintf("dbs.copyGreenToBlue(blue='%v', "+
"green='%v') for index='%v', shard='%v'",
blueDest, greenSrc, idx.name, int(shard)))
}
}
return nil
} // end of fun definition
if !pj.run(fun) {
break indexloop
}
if !firstPjobStarted {
firstPjobStarted = true
defer func() {
err1 := pj.waitForFinish()
if err0 == nil {
err0 = err1
}
}()
}
}
}
return nil
}
func (f *TxFactory) shardSetDiff(blueShards, greenShards map[uint64]bool) (diff string) {
nb := len(blueShards)
ng := len(greenShards)
if nb != ng {
diff = fmt.Sprintf("blueShard[%v] count = %v; greenShard[%v] count = %v; ", f.types[0], nb, f.types[1], ng)
}
bmg := mapDiff(blueShards, greenShards) // get blue - green
gmb := mapDiff(greenShards, blueShards) // get green - blue
if len(bmg) == 0 && len(gmb) == 0 {
return ""
}
diff += fmt.Sprintf("shard diff: blueMinusGreen shards: '%#v'; greenMinusBlue shards: '%#v'", bmg, gmb)
return
}
func (f *TxFactory) GetDBShardPath(index string, shard uint64, idx *Index, ty txtype, write bool) (shardPath string, err error) {
dbs, err := f.dbPerShard.GetDBShard(index, shard, idx)
if err != nil {

385
txfactory_internal_test.go
View file

@ -15,393 +15,14 @@
package pilosa
import (
"context"
"fmt"
"os"
"testing"
"time"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
)
func Test_TxFactory_Qcx_query_context(t *testing.T) {
src := CurrentBackend()
if src == "rbf" || src == "bolt" {
// ok
} else {
t.Skip("this test only for rbf and bolt")
}
shard := uint64(0)
f, idx, tx := mustOpenFragment(t, "i", "f", viewStandard, shard, "")
defer f.Clean(t)
tx.Rollback()
barrier := NewBarrier()
defer barrier.Close()
done := make(chan bool)
setter := func(k int) {
for i := 0; ; i++ {
barrier.WaitAtGate(0)
select {
case <-done:
return
default:
}
// add to the group txn on the txf.
qcx := idx.holder.txf.NewQcx()
tx, finisher, err := qcx.GetTx(Txo{Write: true, Index: idx, Shard: f.shard})
PanicOn(err)
// Set bits on the fragment.
if _, err := f.setBit(tx, 120, 1); err != nil {
panic(err)
} else if _, err := f.setBit(tx, 120, 6); err != nil {
panic(err)
} else if _, err := f.setBit(tx, 121, 0); err != nil {
panic(err)
}
// should have two containers set in the fragment.
// Verify counts on rows.
if n := f.mustRow(tx, 120).Count(); n != 2 {
panic(fmt.Sprintf("unexpected count: %d", n))
} else if n := f.mustRow(tx, 121).Count(); n != 1 {
panic(fmt.Sprintf("unexpected count: %d", n))
}
finisher(nil) // hit the write tx.Commit path
// commit the change, and verify it is still there
PanicOn(qcx.Finish())
qcx.Reset()
tx, finread, err := qcx.GetTx(Txo{Write: !writable, Index: idx, Fragment: f, Shard: f.shard})
PanicOn(err)
if n := f.mustRow(tx, 120).Count(); n != 2 {
panic(fmt.Sprintf("unexpected count (reopen): %d", n))
} else if n := f.mustRow(tx, 121).Count(); n != 1 {
panic(fmt.Sprintf("unexpected count (reopen): %d", n))
}
finread(nil) // no-op on reads that are in a group, so must qcx.Abort() to stop them.
qcx.Abort()
qcx.Reset()
}
}
N := 1000
for i := 0; i < N; i++ {
go setter(i)
}
time.Sleep(time.Second * 1)
close(done)
// allow all goro to finish before Closing the lmdb.env, otherwise
// we will crash as the goroutines making Tx will try to use the env
// after it is closed. It can take quite a while.
// one writer might be blocking the other... so ask for only N-2 at first
// to avoid deadlock.
barrier.BlockUntil(N - 2)
barrier.UnblockReaders()
time.Sleep(1 * time.Second)
}
// test TxFactory.green2blue
//
// blue_green starting with an empty or full blue database
// should copy all of green (if blue is empty); or if blue is ull,
// verify that blue has all the same bits as green.
//
// Benefits: a) we start with known identical state so our testing/comparisons can be valid;
// and b) we have an easy migration mechanism, to go from one storage format to another.
//
func Test_TxFactory_UpdateBlueFromGreen_OnStartup(t *testing.T) {
checked := []string{"roaring", "rbf"}
expectError := false
for _, blue := range checked {
for _, green := range checked {
if blue == green {
continue
}
if blue == "roaring" {
// not supported
expectError = true
} else {
expectError = false
}
blue_green := blue + "_" + green
//vv("setting blue_green to '%v'", blue_green)
// =============================
// Begin setup.
//
// Setup happens with green only.
h, path, err := makeHolder(t, green)
if err != nil {
t.Fatalf("creating holder: %v", err)
}
defer os.RemoveAll(path)
//vv("path = %v", path)
// we will manually h.Close() below
// Write bits to separate indexes.
testSetBit(t, h, "i0", "f", 100, 200)
testSetBit(t, h, "i1", "f", 100, 200)
testSetBit(t, h, "i1", "f", 100, 12345678)
testOp := testHolderOperator{}
ctx := context.Background()
err = h.Process(ctx, &testOp)
if err != nil {
t.Fatalf("processing holder: %v", err)
}
expected := testHolderOperator{
indexSeen: 2, indexProcessed: 2,
fieldSeen: 2, fieldProcessed: 2,
viewSeen: 2, viewProcessed: 2,
fragmentSeen: 3, fragmentProcessed: 3,
}
if testOp != expected {
t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp)
}
// verify data is there
rowID := uint64(100)
colID := uint64(200)
_, _ = rowID, colID
testMustHaveBit(t, h, "i0", "f", rowID, colID)
testMustHaveBit(t, h, "i1", "f", 100, 200)
testMustHaveBit(t, h, "i1", "f", 100, 12345678)
//vv("about to reopen; blue_green = '%v' but PILOSA_STORAGE_BACKEND='%v'", blue_green, os.Getenv("PILOSA_STORAGE_BACKEND"))
//h.DumpAllShards()
//vv("after dump, about to close")
h.Close()
//vv("after close, about to re-open")
// can we re.Open the same holder h? hopefully without a problem.
PanicOn(h.Open())
//vv("h.Open() re-open worked; blue_green = '%v'; dump; with PILOSA_STORAGE_BACKEND='%v'", blue_green, os.Getenv("PILOSA_STORAGE_BACKEND"))
//h.DumpAllShards()
testMustHaveBit(t, h, "i0", "f", rowID, colID) // panic here, colID 200 bit was cold.
testMustHaveBit(t, h, "i1", "f", 100, 200)
testMustHaveBit(t, h, "i1", "f", 100, 12345678)
h.Close()
//vv("successful re-open and then Close again of h.")
// check that we can open a NewHolder on green, on same path, and still see our bits.
// Because the NewHolder is the code that creates and configures TxFactory as blue_green.
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = green
h2 := NewHolder(path, cfg)
PanicOn(h2.Open())
testMustHaveBit(t, h2, "i0", "f", rowID, colID)
testMustHaveBit(t, h2, "i1", "f", 100, 200)
testMustHaveBit(t, h2, "i1", "f", 100, 12345678)
h2.Close()
// verify that blue does not have it.
// open a new holder on path, just looking at blue.
cfg = mustHolderConfig()
cfg.StorageConfig.Backend = blue
h3 := NewHolder(path, cfg)
PanicOn(h3.Open())
testMustNotHaveBit(t, h3, "i0", "f", rowID, colID)
testMustNotHaveBit(t, h3, "i1", "f", 100, 200)
testMustNotHaveBit(t, h3, "i1", "f", 100, 12345678)
h3.Close()
// =============================
// Setup done. On to actual test.
// Opening in blue_green mode means that once Holder.Open()
// returns without error, the blue and green databases are
// identical.
// Since blue is empty, the blue database will get synched up
// with the green during Holder.Open().
// open a holder with path again, now looking at both blue and green.
// The Holder.Open should do the migration from green, populating blue.
cfg = mustHolderConfig()
cfg.StorageConfig.Backend = blue_green
h4 := NewHolder(path, cfg)
//vv("about to h4.Open we should populate blue from green")
err = h4.Open()
if expectError {
if err == nil {
panic("expected error since migration to roaring not supported")
}
} else {
PanicOn(err)
}
testMustHaveBit(t, h4, "i0", "f", rowID, colID)
testMustHaveBit(t, h4, "i1", "f", 100, 200)
testMustHaveBit(t, h4, "i1", "f", 100, 12345678)
//vv("successfully verified populatingBlueFromGreen with blue_green = '%v'", blue_green)
h4.Close()
os.RemoveAll(path)
}
}
}
// test the situation where we startup blue_green with existing data and
// go to verify it but blue has more data than green.
// That will also cause query divergence.
func Test_TxFactory_verifyBlueEqualsGreen(t *testing.T) {
checked := []string{"roaring", "bolt", "rbf"}
for _, blue := range checked {
for _, green := range checked {
if blue == green {
continue
}
if blue == "roaring" {
// not supported
continue
}
blue_green := blue + "_" + green
// =============================
// Begin setup.
//
// Setup happens with green only.
h, path, err := makeHolder(t, green)
if err != nil {
t.Fatalf("creating holder: %v", err)
}
defer os.RemoveAll(path)
//vv("on green, which is '%v'", green)
// we will manually h.Close() below
// Write bits to separate indexes.
testSetBit(t, h, "i0", "f", 100, 200)
testSetBit(t, h, "i1", "f", 100, 200)
testSetBit(t, h, "i1", "f", 100, 12345678)
testOp := testHolderOperator{}
ctx := context.Background()
err = h.Process(ctx, &testOp)
if err != nil {
t.Fatalf("processing holder: %v", err)
}
expected := testHolderOperator{
indexSeen: 2, indexProcessed: 2,
fieldSeen: 2, fieldProcessed: 2,
viewSeen: 2, viewProcessed: 2,
fragmentSeen: 3, fragmentProcessed: 3,
}
if testOp != expected {
t.Fatalf("holder processor did not process as expected. expected %#v, got %#v", expected, testOp)
}
// verify data is there
rowID := uint64(100)
colID := uint64(200)
_, _ = rowID, colID
testMustHaveBit(t, h, "i0", "f", rowID, colID)
testMustHaveBit(t, h, "i1", "f", 100, 200)
testMustHaveBit(t, h, "i1", "f", 100, 12345678)
h.Close()
// verify that blue does not have it.
// open a new holder on path, just looking at blue.
//vv("on blue, which is '%v'", blue)
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = blue
h3 := NewHolder(path, cfg)
PanicOn(h3.Open())
testMustNotHaveBit(t, h3, "i0", "f", rowID, colID)
testMustNotHaveBit(t, h3, "i1", "f", 100, 200)
testMustNotHaveBit(t, h3, "i1", "f", 100, 12345678)
h3.Close()
// =============================
// Setup done. On to actual test.
// Opening in blue_green mode means that once Holder.Open()
// returns without error, the blue and green databases are
// identical.
// Since blue is empty, the blue database will get synched up
// with the green during Holder.Open().
//vv("on blue_green, which is '%v'", blue_green)
// open a holder with path again, now looking at both blue and green.
// The Holder.Open should do the migration from green, populating blue.
cfg = mustHolderConfig()
cfg.StorageConfig.Backend = blue_green
h4 := NewHolder(path, cfg)
PanicOn(h4.Open())
testMustHaveBit(t, h4, "i0", "f", rowID, colID)
testMustHaveBit(t, h4, "i1", "f", 100, 200)
testMustHaveBit(t, h4, "i1", "f", 100, 12345678)
h4.Close()
// now open just blue, and add a bit to a new index, i2.
//vv("on blue, which is '%v'", blue)
cfg = mustHolderConfig()
cfg.StorageConfig.Backend = blue
h5 := NewHolder(path, cfg)
PanicOn(h5.Open())
testSetBit(t, h5, "i2", "f", 500, 777)
//vv("after adding a bit to blue, we have:")
//h5.DumpAllShards()
h5.Close()
// now open blue_green. should get a verification failure
// due to the extra bit in blue.
// BEGIN verficiation that should ERROR out b/c blue has more data.
// open a holder with path again, now looking at both blue and green.
// The Holder.Open should verify blue against green and notice the extra bit.
cfg = mustHolderConfig()
cfg.StorageConfig.Backend = blue_green
h6 := NewHolder(path, cfg)
err = h6.Open()
//h6.DumpAllShards()
if err == nil {
h6.Close()
t.Fatalf("should have had blue-green verification fail on Holder.Open")
}
h6.Close()
}
}
}
func Test_TxFactory_verifyStringConstantsMatch(t *testing.T) {
// txtype.String() method MUST return strings that match
// our const definitions at the top of txfactory.go, or
// else blue-green transactions cannot determine when
// the second transaction is being released in dbshard.go.
check := []txtype{roaringTxn, rbfTxn, boltTxn}
expect := []string{RoaringTxn, RBFTxn, BoltTxn}
// our const definitions at the top of txfactory.go.
check := []txtype{roaringTxn, rbfTxn}
expect := []string{RoaringTxn, RBFTxn}
for i, chk := range check {
obs := chk.String()
if obs != expect[i] {

230
util.go
View file

@ -17,19 +17,12 @@ package pilosa
// util.go: a place for generic, reusable utilities.
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"sort"
"strings"
"syscall"
"time"
"unsafe"
"github.com/molecula/featurebase/v2/roaring"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
"github.com/pkg/errors"
)
@ -63,229 +56,6 @@ func NilInside(iface interface{}) bool {
func highbits(v uint64) uint64 { return v >> 16 }
func lowbits(v uint64) uint16 { return uint16(v & 0xFFFF) }
func toArray16(a []byte) []uint16 {
return (*[4096]uint16)(unsafe.Pointer(&a[0]))[: len(a)/2 : len(a)/2]
}
func toArray64(a []byte) []uint64 {
return (*[1024]uint64)(unsafe.Pointer(&a[0]))[:1024:1024]
}
func toInterval16(a []byte) []roaring.Interval16 {
return (*[2048]roaring.Interval16)(unsafe.Pointer(&a[0]))[: len(a)/4 : len(a)/4]
}
func sliceToMap(slc []uint64) (m map[uint64]bool) {
m = make(map[uint64]bool)
for _, v := range slc {
m[v] = true
}
return
}
// return A - B
func mapDiff(mapA, mapB map[uint64]bool) (r []int) {
for a := range mapA {
_, ok := mapB[a]
if !ok {
r = append(r, int(a))
}
}
sort.Ints(r)
return
}
func asInts(a []uint64) (r []int) {
r = make([]int, len(a))
for i, v := range a {
r[i] = int(v)
}
return
}
func containerAsString(ckey uint64, rc *roaring.Container) (r string) {
rbm := roaring.NewBitmap()
rbm.Containers.Put(ckey, rc)
return BitmapAsString(rbm)
}
var _ = containerAsString // happy linter
func roaringBitmapDiff(a, b *roaring.Bitmap) error {
nA := a.Count()
nB := b.Count()
slcA := a.Slice()
slcB := b.Slice()
mapA := sliceToMap(slcA)
mapB := sliceToMap(slcB)
AminusB := mapDiff(mapA, mapB)
BminusA := mapDiff(mapB, mapA)
sort.Ints(AminusB)
sort.Ints(BminusA)
res := fmt.Sprintf("nA = %v; nB = %v;\n", nA, nB)
ndiff := 0
if nA != nB {
ndiff++
}
if len(AminusB) > 0 {
res += fmt.Sprintf("==> AminusB = (len %v) '%#v'; ", len(AminusB), AminusB)
ndiff++
}
if len(BminusA) > 0 {
res += fmt.Sprintf("\n==> BminusA = (len %v) '%#v'; ", len(BminusA), BminusA)
ndiff++
}
if ndiff == 0 {
return nil
}
res += fmt.Sprintf("\n ==> A = '%#v'\n ==> B = '%#v'", asInts(slcA), asInts(slcB))
return errors.New(res)
}
func dirAsString(path string) (r string) {
r = fmt.Sprintf("dump of directory '%v':\n", path)
files, err := ioutil.ReadDir(path)
PanicOn(err)
for _, f := range files {
r += f.Name() + "\n"
}
return r
}
var _ = dirAsString // happy linter
var _ = zeroKeyContainerAsString // happy linter
// for debugging
func zeroKeyContainerAsString(ct *roaring.Container) (r string) {
cts := roaring.NewSliceContainers()
cts.Put(0, ct)
rbm := &roaring.Bitmap{Containers: cts}
r = fmt.Sprintf("[%v]:", containerTypeNames[roaring.ContainerType(ct)]) + BitmapAsString(rbm)
return
}
var containerTypeNames = map[byte]string{
roaring.ContainerArray: "array",
roaring.ContainerBitmap: "bitmap",
roaring.ContainerRun: "run",
}
func BitmapAsString(rbm *roaring.Bitmap) (r string) {
r = "c("
slc := rbm.Slice()
width := 0
s := ""
for _, v := range slc {
if width == 0 {
s = fmt.Sprintf("%v", v)
} else {
s = fmt.Sprintf(", %v", v)
}
width += len(s)
r += s
if width > 70 {
r += ",\n"
width = 0
}
}
if width == 0 && len(r) > 2 {
r = r[:len(r)-2]
}
return r + ")"
}
// fromArray16 converts to an 8KB page
func fromArray16(a []uint16) []byte {
if len(a) == 0 {
return []byte{}
}
if len(a) > 4096 {
PanicOn(fmt.Sprintf("cannot put more than 4096 integers into an array container: %v too big", len(a)))
}
return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*2 : len(a)*2]
}
// fromArray64 converts to an 8KB page
func fromArray64(a []uint64) []byte {
if len(a) == 0 {
return []byte{}
}
return (*[8192]byte)(unsafe.Pointer(&a[0]))[:8192:8192]
}
// fromInterval16 converts to 8KB page
func fromInterval16(a []roaring.Interval16) []byte {
if len(a) == 0 {
return []byte{}
}
if len(a) > 2048 {
PanicOn(fmt.Sprintf("cannot put more than 2048 roaring.Interval16 into a container: %v too big", len(a)))
}
return (*[8192]byte)(unsafe.Pointer(&a[0]))[: len(a)*4 : len(a)*4]
}
// DiskUse reports the total bytes uses by all files under root
// that match requiredSuffix. requiredSuffix can be empty string.
// Space used by directories is not counted.
func DiskUse(root string, requiredSuffix string) (tot int, err error) {
if !DirExists(root) {
return -1, fmt.Errorf("listFilesUnderDir error: root directory '%v' not found", root)
}
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if info == nil {
PanicOn(fmt.Sprintf("info was nil for path = '%v'", path))
}
if info.IsDir() {
// skip the size of directories themselves, only summing files.
} else {
sz := info.Size()
if requiredSuffix == "" || strings.HasSuffix(path, requiredSuffix) {
tot += int(sz)
}
}
return nil
})
return
}
// rootDir must exist. Return the size in bytes of the largest sub-directory
// that has the required suffix. The largestSize is from DiskUse() called
// on the sub-dir. DiskUse only counts file size, nothing for directory inodes.
func SubdirLargestDirWithSuffix(rootDir, requiredDirSuffix string) (exists bool, largestSize int, err error) {
if !DirExists(rootDir) {
return false, -1, fmt.Errorf("SubdirExistsWithSuffix error: root directory '%v' not found", rootDir)
}
err = filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
if info == nil {
PanicOn(fmt.Sprintf("info was nil for path = '%v'", path))
}
if info.IsDir() && strings.HasSuffix(path, requiredDirSuffix) {
exists = true
size, err := DiskUse(path, "")
if err != nil {
// disk error? report it
return err
}
if size > largestSize {
largestSize = size
}
}
return nil
})
if err != nil {
return exists, -1, err
}
return
}
// called by Holder.hasRoaringData()
func roaringFragmentHasData(path string, index, field, view string, shard uint64) (hasData bool, err error) {

30
utils_internal_test.go
View file

@ -15,47 +15,17 @@
package pilosa
import (
"bytes"
"fmt"
"testing"
"time"
pnet "github.com/molecula/featurebase/v2/net"
"github.com/molecula/featurebase/v2/roaring"
"github.com/molecula/featurebase/v2/testhook"
"github.com/molecula/featurebase/v2/topology"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
)
// utilities used by tests
// mustAddR is a helper for calling roaring.Container.Add() in tests to
// keep the linter happy that we are checking the error.
func mustAddR(changed bool, err error) {
PanicOn(err)
}
// mustRemove is a helper for calling Tx.Remove() in tests to
// keep the linter happy that we are checking the error.
func mustRemove(changeCount int, err error) {
PanicOn(err)
}
func getTestBitmapAsRawRoaring(bitsToSet ...uint64) []byte {
b := roaring.NewBitmap()
changed := b.DirectAddN(bitsToSet...)
n := len(bitsToSet)
if changed != n {
panic(fmt.Sprintf("changed=%v but bitsToSet len = %v", changed, n))
}
buf := bytes.NewBuffer(make([]byte, 0, 100000))
_, err := b.WriteTo(buf)
if err != nil {
panic(err)
}
return buf.Bytes()
}
// NewTestCluster returns a cluster with n nodes and uses a mod-based hasher.
func NewTestCluster(tb testing.TB, n int) *cluster {
path, err := testhook.TempDir(tb, "pilosa-cluster-")

16
view.go
View file

@ -158,26 +158,28 @@ func (v *view) openWithShardSet(ss *shardSet) error {
if nGoro < 4 {
nGoro = 4
}
pj := newParallelJobs(nGoro)
var eg errgroup.Group
throttle := make(chan struct{}, nGoro)
for i := range frags {
// create a new variable frag on each time through
// the loop (instead of i, frag := range frags)
// so that the closure run on the
// goroutine has its own variable.
frag := frags[i]
accepted := pj.run(func(worker int) error {
throttle <- struct{}{}
eg.Go(func() error {
defer func() {
<-throttle
}()
if err := frag.Open(); err != nil {
return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)
}
return nil
})
if !accepted {
// have error/shutting down the pj, so stop
break
}
}
err := pj.waitForFinish()
err := eg.Wait()
if err != nil {
return err
}