mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 00:55:55 +00:00
97b530ca78
- all tests green on RoaringTx
- RoaringTx on by default
- blueGreenTx testing framework available for A-vs-B comparison
of Tx implementations
- flag -tx added to server command line but not wired to
change NewIndex() selection yet.
- 918 green tests, 14 tests red on BadgerTx.
A full list of the 14 red tests on BadgerTx follows.
Note that these red tests represent not defects in BadgerDB
or BadgerTx but rather failures of the pre-existing pilosa infrastructure to yet
be fully adapted from files to using a transactional storage engine.
As such these are tests that RBF should not be expected to
pass yet either.
Fixing the pilosa infrastructure to allow these tests
to go green under Badger is the next and highest priority
order of business, but RBF can get much testing benefit
from the 918 green tests we do have, and hence we merge
as much as we have today.
The 14 red tests when NewIndex() is set to use
BadgerTx are as follows. Note in particular
that pilosa cluster resizing is not working yet under a
transactional store.
TestCluster_ResizeStates/Multiple_nodes,_with_data
TestImportClearRestart/0MaxOpN10000
TestImportClearRestart/1MaxOpN10000
TestImportClearRestart/2MaxOpN10000
TestImportClearRestart/3MaxOpN10000
TestExecutor_Execute_Existence/Row
TestExecutor_ForeignIndex
TestExecutor_Execute_CountDistinct/Distinct
TestExecutor_Execute_CountDistinct/Count(Distinct)
TestExecutor_Execute_CountDistinct/GroupBy(Distinct)
TestExecutor_BareDistinct
TestExecutor_Execute_TopNDistinct/TopN
TestHolderSyncer_IntField/BasicSync
TestHolderSyncer_IntField/MultiShard
105 lines
2.8 KiB
Go
105 lines
2.8 KiB
Go
// Copyright 2020 Pilosa Corp.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package pilosa
|
|
|
|
import (
|
|
"fmt"
|
|
"math/rand"
|
|
"runtime"
|
|
"testing"
|
|
|
|
"github.com/pilosa/pilosa/v2/logger"
|
|
"github.com/pilosa/pilosa/v2/syswrap"
|
|
)
|
|
|
|
type cv struct {
|
|
cols []uint64
|
|
vals []int64
|
|
}
|
|
|
|
func forceSnapshotsCheckMapping(t *testing.T) {
|
|
depth := uint(6)
|
|
f, idx := mustOpenBSIFragment("i", "f", viewStandard, 0)
|
|
_ = idx
|
|
f.Logger = logger.NewLogfLogger(t)
|
|
defer f.Clean(t)
|
|
|
|
tx := &RoaringTx{fragment: f}
|
|
|
|
for i := 0; i < f.MaxOpN; i++ {
|
|
_, _ = f.setBit(tx, 0, uint64(32*i))
|
|
}
|
|
// force snapshot so we get a mmapped row...
|
|
err := f.Snapshot()
|
|
if err != nil {
|
|
t.Fatalf("initial snapshot error: %v", err)
|
|
}
|
|
|
|
values := make([]cv, 1024)
|
|
for i := range values {
|
|
cols := make([]uint64, 128)
|
|
vals := make([]int64, 128)
|
|
for j := range cols {
|
|
// pick values in the first 16 cols of each of the 16
|
|
// shards in a default shardwidth, so each set will
|
|
// probably change some values from the previous one.
|
|
cols[j] = uint64(((rand.Int63n(16) & int64(i>>2)) << 16) + rand.Int63n(16))
|
|
vals[j] = int64(rand.Int63n(1 << depth))
|
|
}
|
|
values[i] = cv{cols, vals}
|
|
}
|
|
|
|
// modify the original bitmap, until it causes a snapshot, which
|
|
// then invalidates the other map...
|
|
for i := 0; i < 32; i++ {
|
|
cv := values[i%len(values)]
|
|
// periodically force gc, so if we have a small pool of maps
|
|
// we'll go in and out of mapping mode
|
|
if i%5 == 0 {
|
|
runtime.GC()
|
|
}
|
|
err := f.importValue(tx, cv.cols, cv.vals, depth, (i%3 == 1))
|
|
if err != nil {
|
|
t.Fatalf("importValue[%d]: %v", i, err)
|
|
}
|
|
err = f.Snapshot()
|
|
if err != nil {
|
|
t.Fatalf("snapshot[%d]: %v", i, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// This test should basically never fail, but it might if you were running
|
|
// out of available mmaps. Which you can fake up by adding '&& false' to the test
|
|
// in newGeneration in generation.go. So this is probably useless but it's
|
|
// a failure mode we've been bitten by once...
|
|
func TestMmapBehavior(t *testing.T) {
|
|
var changed bool
|
|
var original uint64
|
|
defer func() {
|
|
syswrap.SetMaxMapCount(original)
|
|
}()
|
|
|
|
for _, mmapMaxVal := range []uint64{0, 3} {
|
|
prev := syswrap.SetMaxMapCount(mmapMaxVal)
|
|
if !changed {
|
|
original = prev
|
|
changed = true
|
|
}
|
|
t.Run(fmt.Sprintf("maps%d", mmapMaxVal), func(t *testing.T) {
|
|
forceSnapshotsCheckMapping(t)
|
|
})
|
|
}
|
|
}
|