mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-06 00:25:55 +00:00
ad30a926f4
These commits are hard to disentagle, and doing them separately means re-modifying the same chunks of code several times before removing it, and similar things. Basically: (1) Drop the bolt backend storage. (2) Drop the blue-green wrapper that compares two backends. (3) Drop unused or barely-used Tx API components from all the remaining backends. (4) Minor related cleanup to simplify things related to these. The boltdb backend existed only to verify RBF. The blue-green wrapper was mostly used to verify RBF, but in practice we had to do a lot of working around that, and it introduced a lot of special cases. Types removed: IteratorFinder: Used only to implement the roaring iterator on top of boltdb, and to complicate the way it worked in roaring. Reverted the complications. Also unexport NewSliceContainers which is used only for that outside of roaring's internals. PortMapper from cluster_internal_test.go: Used only for a test we removed early this year. Never used for anything else. RawRoaringData: Totally unused. TxStore: Totally unused. Functions removed from Tx API, and sometimes corresponding members were removed from structs: * Dump: debugging code, I don't think I found any actually reachable paths to it. * Group: only used for debugging TxGroup stuff * IncrementOpN: only used by fragment, fragment can increment its own opN. * Options: unused? * Pointer: debugging only * Readonly: used only to decide how to handle Tx in a TxGrp, but we never add a non-readonly Tx to a TxGrp. Removed also all the corresponding write-aware stuff. * RoaringBitmapReader: Used exactly once, can just be a bm.WriteTo. * Sn (and OpenSnList): Unused * UnionInPlace: unused and conceptually-invalid; it didn't write to storage and shouldn't have, and was just "create a bitmap then call union-in-place", which we can do directly. * UseRowCache: just checked storage.UseRowCache. Other things removed: The SetRequiredForAtomicWriteTx and ClearRequiredForAtomicWriteTx functions go away, since nothing now seems to be using them? Same for holder_internal_test's `testHasBit` and `testMustNotHaveBit`, which were unused. The DBPerShard "DeleteDBPath" and "HasData" functions and related parts were mostly unused; took out the parts that were never actually being reached. Changed the API of one function to simplify special cases and remove things: * ImportRoaringBits had a special "data" argument which gave it subtly different semantics for RBF and roaring (for roaring, it could produce a roaring bitmap *with ops log*), didn't seem to be adding much. Removed corresponding "readStorageFromArchive" which is not otherwise used. Also took out various debugging/dumping functions that were unused and may have bitrotted. Dropped a test from txfactory_internal_test, and the "pjobs" code, because those two were the only things that needed Barrier and thus idem, which lets us drop two more dependencies. We already have errgroup for grouping things which want to terminate as soon as one of them errors, approximately. To do better we'd have to have context-threading, really. Unbroke the WriteFragment test for non-roaring tests and made it not roaring-only.
519 lines
12 KiB
Go
519 lines
12 KiB
Go
// Copyright 2017 Pilosa Corp.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package rbf_test
|
|
|
|
import (
|
|
"fmt"
|
|
"math/rand"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/molecula/featurebase/v2/rbf"
|
|
)
|
|
|
|
func TestTx_CommitRollback(t *testing.T) {
|
|
t.Run("NoReopen", func(t *testing.T) {
|
|
db := MustOpenDB(t)
|
|
defer MustCloseDB(t, db)
|
|
|
|
// Create bitmap in transaction but rollback.
|
|
tx, err := db.Begin(true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tx.Rollback()
|
|
|
|
// Create bitmap in transaction again but commit.
|
|
if tx, err := db.Begin(true); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.Commit(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Create bitmap again but it should fail as it already exists.
|
|
if tx, err := db.Begin(true); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.CreateBitmap("x"); err == nil || err != rbf.ErrBitmapExists {
|
|
tx.Rollback()
|
|
t.Fatal(err)
|
|
} else if err := tx.Commit(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
t.Run("Reopen", func(t *testing.T) {
|
|
db := MustOpenDB(t)
|
|
defer func() { MustCloseDB(t, db) }()
|
|
|
|
// Create bitmap in transaction but rollback.
|
|
tx, err := db.Begin(true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tx.Rollback()
|
|
db = MustReopenDB(t, db)
|
|
|
|
// Create bitmap in transaction again but commit.
|
|
if tx, err := db.Begin(true); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.Commit(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
db = MustReopenDB(t, db)
|
|
|
|
// Create bitmap again but it should fail as it already exists.
|
|
if tx, err := db.Begin(true); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.CreateBitmap("x"); err == nil || err != rbf.ErrBitmapExists {
|
|
tx.Rollback()
|
|
t.Fatal(err)
|
|
} else if err := tx.Commit(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
t.Run("ReopenReadOnly", func(t *testing.T) {
|
|
db := MustOpenDB(t)
|
|
defer func() { MustCloseDB(t, db) }()
|
|
|
|
// Create bitmap in transaction and commit.
|
|
if tx, err := db.Begin(true); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.Commit(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
db = MustReopenDB(t, db)
|
|
|
|
// Create bitmap again but it should fail as it already exists.
|
|
if tx, err := db.Begin(false); err != nil {
|
|
t.Fatal(err)
|
|
} else if _, err := tx.Count("x"); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.Commit(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
|
|
t.Run("SingleWriter", func(t *testing.T) {
|
|
db := MustOpenDB(t)
|
|
defer MustCloseDB(t, db)
|
|
|
|
var wg sync.WaitGroup
|
|
defer wg.Wait()
|
|
|
|
// Start write transaction.
|
|
ch0 := make(chan struct{})
|
|
tx0 := MustBegin(t, db, true)
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
<-ch0
|
|
tx0.Rollback()
|
|
}()
|
|
|
|
// Start separate write transaction in different goroutine.
|
|
ch1 := make(chan struct{})
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
tx1 := MustBegin(t, db, true)
|
|
close(ch1)
|
|
_ = tx1.Commit()
|
|
}()
|
|
|
|
// Ensure second tx doesn't start.
|
|
select {
|
|
case <-ch1:
|
|
t.Fatal("second tx started while first tx active")
|
|
case <-time.After(10 * time.Millisecond):
|
|
}
|
|
|
|
// Finish first transaction.
|
|
close(ch0)
|
|
select {
|
|
case <-ch1:
|
|
case <-time.After(10 * time.Millisecond):
|
|
t.Fatal("second tx should have started after first tx closed")
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestTx_Add(t *testing.T) {
|
|
db := MustOpenDB(t)
|
|
defer MustCloseDB(t, db)
|
|
tx := MustBegin(t, db, true)
|
|
defer tx.Rollback()
|
|
|
|
if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if _, err := tx.Add("x", 1); err != nil {
|
|
t.Fatal(err)
|
|
} else if _, err := tx.Add("x", 10); err != nil {
|
|
t.Fatal(err)
|
|
} else if _, err := tx.Add("x", 3); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
for _, v := range []uint64{1, 3, 10} {
|
|
if ok, err := tx.Contains("x", v); err != nil {
|
|
t.Fatal(err)
|
|
} else if !ok {
|
|
t.Fatalf("Tx.Contains(%d): expected true", v)
|
|
}
|
|
}
|
|
|
|
if ok, err := tx.Contains("x", 2); err != nil {
|
|
t.Fatal(err)
|
|
} else if ok {
|
|
t.Fatal("Tx.Contains(): expected false")
|
|
}
|
|
}
|
|
|
|
func TestTx_DeleteBitmap(t *testing.T) {
|
|
db := MustOpenDB(t)
|
|
defer MustCloseDB(t, db)
|
|
tx := MustBegin(t, db, true)
|
|
defer tx.Rollback()
|
|
|
|
// Create bitmap & add value.
|
|
if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
} else if _, err := tx.Add("x", 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Recreate bitmap & ensure value does not exist.
|
|
if err := tx.DeleteBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
} else if ok, err := tx.Contains("x", 1); err != nil {
|
|
t.Fatal(err)
|
|
} else if ok {
|
|
t.Fatal("expected no value in recreated bitmap")
|
|
}
|
|
}
|
|
|
|
func TestTx_RenameBitmap(t *testing.T) {
|
|
db := MustOpenDB(t)
|
|
defer MustCloseDB(t, db)
|
|
tx := MustBegin(t, db, true)
|
|
defer tx.Rollback()
|
|
|
|
// Create bitmap & add value.
|
|
if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
} else if _, err := tx.Add("x", 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Rename bitmap & ensure value still exists.
|
|
if err := tx.RenameBitmap("x", "y"); err != nil {
|
|
t.Fatal(err)
|
|
} else if ok, err := tx.Contains("y", 1); err != nil {
|
|
t.Fatal(err)
|
|
} else if !ok {
|
|
t.Fatal("expected value in renamed bitmap")
|
|
}
|
|
}
|
|
|
|
func TestTx_Add_Quick(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("-short enabled, skipping")
|
|
} else if is32Bit() {
|
|
t.Skip("32-bit build, skipping quick check tests")
|
|
}
|
|
|
|
QuickCheck(t, func(t *testing.T, rand *rand.Rand) {
|
|
t.Parallel()
|
|
|
|
db := MustOpenDB(t)
|
|
defer MustCloseDB(t, db)
|
|
tx := MustBegin(t, db, true)
|
|
defer tx.Rollback()
|
|
values := GenerateValues(rand, 10000)
|
|
|
|
if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Insert values in random order.
|
|
for _, i := range rand.Perm(len(values)) {
|
|
v := values[i]
|
|
if _, err := tx.Add("x", v); err != nil {
|
|
t.Fatalf("Add(%d) i=%d err=%q", v, i, err)
|
|
}
|
|
}
|
|
|
|
// Verify all bits are written.
|
|
for i, v := range values {
|
|
if ok, err := tx.Contains("x", v); !ok || err != nil {
|
|
t.Fatalf("Contains(%d)=(%v,%v) i=%d hi=%d lo=%d", v, ok, err, i, highbits(v), lowbits(v))
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestTx_AddRemove_Quick(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("-short enabled, skipping")
|
|
} else if is32Bit() {
|
|
t.Skip("32-bit build, skipping quick check tests")
|
|
}
|
|
|
|
QuickCheck(t, func(t *testing.T, rand *rand.Rand) {
|
|
t.Parallel()
|
|
|
|
db := MustOpenDB(t)
|
|
defer MustCloseDB(t, db)
|
|
tx := MustBegin(t, db, true)
|
|
defer tx.Rollback()
|
|
values := GenerateValues(rand, 10000)
|
|
|
|
if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Insert values in random order.
|
|
for _, i := range rand.Perm(len(values)) {
|
|
if _, err := tx.Add("x", values[i]); err != nil {
|
|
t.Fatalf("Add(%d) i=%d err=%q", values[i], i, err)
|
|
}
|
|
}
|
|
|
|
// Remove half the values in random order.
|
|
for _, i := range rand.Perm(len(values)) {
|
|
if _, err := tx.Remove("x", values[i]); err != nil {
|
|
t.Fatalf("Remove(%d) i=%d err=%q", values[i], i, err)
|
|
}
|
|
}
|
|
|
|
// Verify all bits are removed.
|
|
for i, v := range values {
|
|
if ok, err := tx.Contains("x", v); ok || err != nil {
|
|
t.Fatalf("Contains(%d)=(%v,%v) i=%d hi=%d lo=%d", v, ok, err, i, highbits(v), lowbits(v))
|
|
}
|
|
}
|
|
|
|
// Re-add those values back in.
|
|
for _, i := range rand.Perm(len(values)) {
|
|
if _, err := tx.Add("x", values[i]); err != nil {
|
|
t.Fatalf("Re-Add(%d) i=%d err=%q", values[i], i, err)
|
|
}
|
|
}
|
|
|
|
// Verify all bits are written.
|
|
for i, v := range values {
|
|
if ok, err := tx.Contains("x", v); !ok || err != nil {
|
|
t.Fatalf("Contains(%d)=(%v,%v) i=%d hi=%d lo=%d", v, ok, err, i, highbits(v), lowbits(v))
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestTx_Multiple_CreateBitmap(t *testing.T) {
|
|
rand := rand.New(rand.NewSource(0))
|
|
db := MustOpenDB(t)
|
|
defer MustCloseDB(t, db)
|
|
tx := MustBegin(t, db, true)
|
|
defer tx.Rollback()
|
|
values := GenerateValues(rand, 2)
|
|
|
|
if err := tx.CreateBitmap("x/1"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Insert values in random order.
|
|
for _, i := range rand.Perm(len(values)) {
|
|
if _, err := tx.Add("x/1", values[i]); err != nil {
|
|
t.Fatalf("Add(%d) i=%d err=%q", values[i], i, err)
|
|
}
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
t.Fatalf("Commit 1 err=%q", err)
|
|
}
|
|
|
|
tx1 := MustBegin(t, db, true)
|
|
defer tx1.Rollback()
|
|
|
|
if err := tx1.CreateBitmap("x/2"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Insert values in random order.
|
|
for _, i := range rand.Perm(len(values)) {
|
|
if _, err := tx1.Add("x/2", values[i]); err != nil {
|
|
t.Fatalf("Add(%d) i=%d err=%q", values[i], i, err)
|
|
}
|
|
}
|
|
if err := tx1.Commit(); err != nil {
|
|
t.Fatalf("Commit 2 err=%q", err)
|
|
}
|
|
}
|
|
|
|
func TestTx_CursorCrashArray(t *testing.T) {
|
|
db := MustOpenDB(t)
|
|
defer MustCloseDB(t, db)
|
|
tx := MustBegin(t, db, true)
|
|
defer tx.Rollback()
|
|
if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c, err := tx.Cursor("x")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
//setArray(t, 0, 2379, &c)
|
|
//setArray(t, 1, 2337, &c)
|
|
setArray(t, 32, 1216, c)
|
|
setArray(t, 33, 1195, c)
|
|
setArray(t, 48, 1186, c)
|
|
setArray(t, 49, 1223, c)
|
|
setArray(t, 50, 1223, c)
|
|
|
|
}
|
|
|
|
func TestTx_CursorCrashBitmap(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("-short enabled, skipping")
|
|
}
|
|
|
|
db := MustOpenDB(t)
|
|
defer MustCloseDB(t, db)
|
|
tx := MustBegin(t, db, true)
|
|
defer tx.Rollback()
|
|
if err := tx.CreateBitmap("x"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c, err := tx.Cursor("x")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
setArray(t, 0, 22510, c)
|
|
setArray(t, 1, 23584, c)
|
|
}
|
|
|
|
func setArray(tb testing.TB, key, num int, c *rbf.Cursor) {
|
|
for i := uint64(0); i < uint64(num); i++ {
|
|
v := i | (uint64(key) << 16)
|
|
if _, err := c.Add(v); err != nil {
|
|
tb.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func BenchmarkTx_Add(b *testing.B) {
|
|
for _, n := range []int{1, 10, 1000} {
|
|
b.Run(fmt.Sprint(n), func(b *testing.B) {
|
|
rand := rand.New(rand.NewSource(0))
|
|
|
|
values := make([]uint64, n)
|
|
for i := range values {
|
|
values[i] = uint64(rand.Intn(rbf.ShardWidth))
|
|
}
|
|
b.ResetTimer()
|
|
|
|
b.ReportAllocs()
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
func() {
|
|
db := MustOpenDB(b)
|
|
defer MustCloseDB(b, db)
|
|
|
|
for _, v := range values {
|
|
func() {
|
|
tx := MustBegin(b, db, true)
|
|
defer tx.Rollback()
|
|
if _, err := tx.Add("x", v); err != nil {
|
|
b.Fatal(err)
|
|
} else if err := tx.Commit(); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}()
|
|
}
|
|
}()
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func BenchmarkTx_Contains(b *testing.B) {
|
|
for _, n := range []int{10000, 100000, 1000000} {
|
|
b.Run(fmt.Sprint(n), func(b *testing.B) {
|
|
rand := rand.New(rand.NewSource(0))
|
|
|
|
values := make([]uint64, n)
|
|
for i := range values {
|
|
values[i] = uint64(rand.Intn(rbf.ShardWidth))
|
|
}
|
|
|
|
db := MustOpenDB(b)
|
|
defer MustCloseDB(b, db)
|
|
tx := MustBegin(b, db, true)
|
|
defer tx.Rollback()
|
|
|
|
b.ResetTimer()
|
|
t := time.Now()
|
|
b.ReportAllocs()
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
for _, v := range values {
|
|
if _, err := tx.Contains("x", v); err != nil {
|
|
b.Fatalf("Contains(%d) i=%d err=%q", v, i, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
b.ReportMetric(float64(time.Since(t).Nanoseconds())/float64(n*b.N), "ns/op")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTx_CreateBitmap(t *testing.T) {
|
|
t.Run("Bulk", func(t *testing.T) {
|
|
db := MustOpenDB(t)
|
|
defer MustCloseDB(t, db)
|
|
|
|
tx, err := db.Begin(true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer tx.Rollback()
|
|
|
|
if err := tx.CreateBitmap(fmt.Sprintf("%4000x", 0)); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.CreateBitmap(fmt.Sprintf("%4000x", 1)); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := tx.CreateBitmap(fmt.Sprintf("%4000x", 2)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
}
|