scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into cpumhz

Browse source
This commit is contained in:
Jaden Weiss 2020-06-08 13:26:49 -04:00 • committed by GitHub
commit a4e53b4bc1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 294 additions and 240 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
cluster.go
View file

@ -62,9 +62,8 @@ const (
resizeJobActionAdd = "ADD"
resizeJobActionRemove = "REMOVE"
confirmDownRetries = 10
confirmDownSleep = 1
confirmDownTimeout = 2
defaultConfirmDownRetries = 10
defaultConfirmDownSleep = 1 * time.Second
)
// Node represents a node in the cluster.
@ -239,6 +238,9 @@ type cluster struct { // nolint: maligned
logger logger.Logger
InternalClient InternalClient
confirmDownRetries int
confirmDownSleep time.Duration
}
// newCluster returns a new instance of Cluster with defaults.
@ -258,6 +260,9 @@ func newCluster() *cluster {
InternalClient: newNopInternalClient(),
logger: logger.NopLogger,
confirmDownRetries: defaultConfirmDownRetries,
confirmDownSleep: defaultConfirmDownSleep,
}
}
@ -1923,7 +1928,7 @@ func (c *cluster) considerTopology() error {
// band aid to protect against false nodeLeave events from memberlist
// the test is the lightest weight endpoint of the node in question /version
// TODO provide more robust solution to false nodeLeave events
func confirmNodeDown(uri URI, log logger.Logger) bool {
func (c *cluster) confirmNodeDown(uri URI) bool {
u := url.URL{
Scheme: uri.Scheme,
Host: uri.HostPort(),
@ -1931,11 +1936,11 @@ func confirmNodeDown(uri URI, log logger.Logger) bool {
}
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
log.Printf("bad request:%s %s", u.String(), err)
c.logger.Printf("bad request:%s %s", u.String(), err)
return false
}
for i := 0; i < confirmDownRetries; i++ {
ctx, cancel := context.WithTimeout(context.Background(), confirmDownTimeout*time.Second)
for i := 0; i < c.confirmDownRetries; i++ {
ctx, cancel := context.WithTimeout(context.Background(), c.confirmDownSleep*2)
defer cancel()
resp, err := http.DefaultClient.Do(req.WithContext(ctx))
var bod []byte
@ -1946,8 +1951,8 @@ func confirmNodeDown(uri URI, log logger.Logger) bool {
}
}
log.Printf("NodeLeave confirm with %s %d. err: '%v' bod: '%s'", uri.HostPort(), i, err, bod)
time.Sleep(confirmDownSleep * time.Second)
c.logger.Printf("NodeLeave confirm with %s %d. err: '%v' bod: '%s'", uri.HostPort(), i, err, bod)
time.Sleep(c.confirmDownSleep)
}
return true
}
@ -1975,7 +1980,7 @@ func (c *cluster) ReceiveEvent(e *NodeEvent) (err error) {
// not already removed by a removeNode request. We treat this as the
// host being temporarily unavailable, and expect it to come back
// up.
if confirmNodeDown(e.Node.URI, c.logger) {
if c.confirmNodeDown(e.Node.URI) {
if c.removeNodeBasicSorted(e.Node.ID) {
c.Topology.nodeStates[e.Node.ID] = nodeStateDown
// put the cluster into STARTING if we've lost a number of nodes

21
cluster_internal_test.go
View file

@ -943,18 +943,22 @@ func TestCluster_confirmNodeDownUp(t *testing.T) {
t.Error(err)
}
uri.Port = uint16(iport)
if confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
c := newCluster()
c.logger = logger.NewVerboseLogger(os.Stdout)
if c.confirmNodeDown(uri) {
t.Errorf("expected node to be up")
}
}
func TestCluster_confirmNodeDownTimeout(t *testing.T) {
sleep := 50 * time.Millisecond
retries := 5
if testing.Short() {
t.Skip()
}
r := mux.NewRouter()
r.HandleFunc("/version", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(confirmDownSleep * time.Second * confirmDownRetries)
time.Sleep(sleep * time.Duration(retries))
fmt.Fprintln(w, "ignored")
}))
server := httptest.NewServer(r)
@ -973,8 +977,11 @@ func TestCluster_confirmNodeDownTimeout(t *testing.T) {
t.Error(err)
}
uri.Port = uint16(iport)
if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
c := newCluster()
c.confirmDownSleep = sleep
c.confirmDownRetries = retries
c.logger = logger.NewVerboseLogger(os.Stdout)
if !c.confirmNodeDown(uri) {
t.Errorf("expected node to be down")
}
}
@ -987,8 +994,12 @@ func TestCluster_confirmNodeDownDown(t *testing.T) {
uri.Scheme = "http"
uri.Host = "DoesntMatter"
uri.Port = 6666
c := newCluster()
c.confirmDownSleep = 50 * time.Millisecond
c.confirmDownRetries = 5
c.logger = logger.NewVerboseLogger(os.Stdout)
if !confirmNodeDown(uri, logger.NewVerboseLogger(os.Stdout)) {
if !c.confirmNodeDown(uri) {
t.Errorf("expected node to be down")
}
}

18
executor_test.go
View file

@ -3476,15 +3476,15 @@ func TestExecutor_Execute_All(t *testing.T) {
t.Fatal(err)
}
// Create an import request that sets a full shard,
// plus a couple bits set on either side of it, and
// a final bit set in a fourth shard.
// Create an import request that sets things on either end
// of a shard, plus a couple bits set on either side of it,
// and a final bit set in a fourth shard.
//
// shard0 shard1 shard2 shard3
// |----------|----------|----------|----------|
// | **|**********|** | *
// | **|** **|** | *
//
bitCount := ShardWidth + 5
bitCount := 100 + 5
req := &pilosa.ImportRequest{
Index: index.Name(),
Field: fld.Name(),
@ -3492,10 +3492,14 @@ func TestExecutor_Execute_All(t *testing.T) {
RowIDs: make([]uint64, bitCount),
ColumnIDs: make([]uint64, bitCount),
}
for i := 0; i < bitCount-1; i++ {
for i := 0; i < bitCount/2; i++ {
req.RowIDs[i] = 10
req.ColumnIDs[i] = uint64(i + ShardWidth - 2)
}
for i := bitCount / 2; i < bitCount-1; i++ {
req.RowIDs[i] = 10
req.ColumnIDs[i] = uint64(i + (ShardWidth * 2) - bitCount + 5)
}
req.RowIDs[bitCount-1] = 10
req.ColumnIDs[bitCount-1] = uint64((3 * ShardWidth) + 2)
@ -3521,7 +3525,7 @@ func TestExecutor_Execute_All(t *testing.T) {
{qry: fmt.Sprintf("All(limit=2, offset=%d)", bitCount-5), expCols: req.ColumnIDs[bitCount-5 : bitCount-3], expCnt: 2},
{qry: "All(limit=2, offset=2)", expCols: req.ColumnIDs[2:4], expCnt: 2},
{qry: "All(limit=1, offset=1)", expCols: req.ColumnIDs[1:2], expCnt: 1},
{qry: fmt.Sprintf("All(limit=%d, offset=2)", ShardWidth), expCols: req.ColumnIDs[2 : bitCount-3], expCnt: ShardWidth},
{qry: fmt.Sprintf("All(limit=%d, offset=2)", bitCount-3), expCols: req.ColumnIDs[2 : bitCount-1], expCnt: uint64(bitCount - 3)},
}
for i, test := range tests {
if res, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: test.qry}); err != nil {

9
fragment_internal_test.go
View file

@ -111,6 +111,11 @@ func TestFragment_ClearBit(t *testing.T) {
func TestFragment_RowcacheMap(t *testing.T) {
var done int64
f := mustOpenFragment("i", "f", viewStandard, 0, "")
// Under -race, this test turns out to take a fairly long time
// to run with larger OpN, because we write 50,000 bits to
// the bitmap, and everything is being race-detected, and we don't
// actually need that many to get the result we care about.
f.MaxOpN = 2000
defer f.Clean(t)
ch := make(chan struct{})
@ -2755,9 +2760,9 @@ func TestFragment_RowsIteration(t *testing.T) {
defer f.Clean(t)
expectedRows := make([]uint64, 0)
for r := uint64(1); r < uint64(10000); r += 100 {
for r := uint64(1); r < uint64(10000); r += 250 {
expectedRows = append(expectedRows, r)
for c := uint64(1); c < uint64(ShardWidth-1); c += 10000 {
for c := uint64(1); c < uint64(ShardWidth-1); c += (ShardWidth >> 5) {
if _, err := f.setBit(r, c); err != nil {
t.Fatal(err)
}

6
roaring/btree_test.go
View file

@ -689,7 +689,7 @@ func TestBtreeDelete0(t *testing.T) {
}
func TestBtreeDelete1(t *testing.T) {
const N = 130000
const N = 13000
for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x333333, 0xcccccc, 0x314159} {
r := treeNew()
set := r.Set
@ -788,7 +788,7 @@ func benchmarkDelRnd(b *testing.B, n int) {
}
func TestBtreeDelete2(t *testing.T) {
const N = 100000
const N = 10000
for _, x := range []int{0, -1, 0x555555, 0xaaaaaa, 0x333333, 0xcccccc, 0x314159} {
r := treeNew()
set := r.Set
@ -1468,7 +1468,7 @@ func TestBtreePut(t *testing.T) {
}
func TestBtreeSeek(t *testing.T) {
const N = 1 << 13
const N = 1 << 11
tr := treeNew()
for i := 0; i < N; i++ {
k := 2*i + 1

5
roaring/naive_test.go
View file

@ -15,6 +15,7 @@
package roaring
import (
"math/rand"
"reflect"
"testing"
)
@ -110,13 +111,15 @@ func TestUnionSlice(t *testing.T) {
}
func TestMaxInSlice(t *testing.T) {
// arbitrary, we just want to get the same values every time
r := rand.New(rand.NewSource(23))
a := []uint64{1, 4, 9, 5, 24, 13}
v := maxInSlice(a)
if uint64(24) != v {
t.Fatalf("expected %v, but got %v", uint64(24), v)
}
for i := uint64(1000); i <= uint64(100000); i++ {
for i := uint64(1000); i <= uint64(100000); i += uint64(r.Intn(35)) + 1 {
a = append(a, i)
if v = maxInSlice(a); v != i {
t.Fatalf("expected %v, but got %v", i, v)

186
roaring/roaring.go
View file

@ -4552,50 +4552,95 @@ func (c *Container) bitmapZeroRange(i, j uint64) {
c.setN(n)
}
// equals reports whether two containers are equal.
func (c *Container) equals(c2 *Container) bool {
if c == nil || c2 == nil {
if c != c2 {
return false
func typePair(ct1, ct2 byte) int {
return int((ct1 << 4) | ct2)
}
// compareArrayBitmap actually only verifies that everything in the array
// is in the bitmap. It's used only after comparing the N for the containers,
// so if there's anything in the bitmap that's not in the array, either there's
// something in the array that's not in the bitmap, or we didn't get here.
func compareArrayBitmap(a []uint16, b []uint64) error {
for _, v := range a {
w, bit := b[v>>6], v&63
if w>>bit&1 == 0 {
return fmt.Errorf("value %d missing", v)
}
}
if c.Mapped() != c2.Mapped() || c.typ() != c2.typ() || c.N() != c2.N() {
return false
return nil
}
// compareArrayRuns determines whether an array matches a provided
// set of runs. As with compareArrayBitmap, it only verifies presence
// of the array's values in the run collection. the run collection
// can't be empty; if it were, N would have been 0, and we wouldn't
// have gotten here.
func compareArrayRuns(a []uint16, r []interval16) error {
ri := 0
ru := r[ri]
ri++
for _, v := range a {
if v < ru.start {
return fmt.Errorf("value %d missing", v)
}
if v > ru.last {
if ri >= len(r) {
return fmt.Errorf("value %d missing", v)
}
ru = r[ri]
ri++
// if they're identical, the array value must be
// the start of the next run.
if v != ru.start {
return fmt.Errorf("value %d missing", v)
}
}
}
if c.typ() == containerArray {
ca, c2a := c.array(), c2.array()
if len(ca) != len(c2a) {
return false
}
for i := 0; i < len(ca); i++ {
if ca[i] != c2a[i] {
return false
}
}
} else if c.typ() == containerBitmap {
cb, c2b := c.bitmap(), c2.bitmap()
if len(cb) != len(c2b) {
return false
}
for i := 0; i < len(cb); i++ {
if cb[i] != c2b[i] {
return false
}
}
} else if c.typ() == containerRun {
cr, c2r := c.runs(), c2.runs()
if len(cr) != len(c2r) {
return false
}
for i := 0; i < len(cr); i++ {
if cr[i] != c2r[i] {
return false
}
}
} else {
panic(fmt.Sprintf("unknown container type: %v", c.typ()))
return nil
}
// compareArrayArray reports whether everything in a1 is equal to everything
// in a2.
func compareArrayArray(a1, a2 []uint16) error {
if len(a1) != len(a2) {
return fmt.Errorf("unexpected length mismatch, %d vs %d", len(a1), len(a2))
}
return true
for i := range a1 {
if a1[i] != a2[i] {
return fmt.Errorf("item %d: %d vs %d", i, a1[i], a2[i])
}
}
return nil
}
// BitwiseCompare reports whether two containers are equal. It returns
// an error describing any difference it finds. This is mostly intended
// for use in tests that expect equality.
func (c *Container) BitwiseCompare(c2 *Container) error {
if c.N() != c2.N() {
return errors.New("containers are different lengths")
}
if c.N() == 0 {
return nil
}
switch typePair(c.typ(), c2.typ()) {
case typePair(containerArray, containerArray):
return compareArrayArray(c.array(), c2.array())
case typePair(containerArray, containerBitmap):
return compareArrayBitmap(c.array(), c2.bitmap())
case typePair(containerBitmap, containerArray):
return compareArrayBitmap(c2.array(), c.bitmap())
case typePair(containerArray, containerRun):
return compareArrayRuns(c.array(), c2.runs())
case typePair(containerRun, containerArray):
return compareArrayRuns(c2.array(), c.runs())
default:
c3 := xor(c, c2)
if c3.N() != 0 {
return fmt.Errorf("%d bits differenct between containers", c3.N())
}
}
return nil
}
func unionArrayBitmap(a, b *Container) *Container {
@ -5756,7 +5801,37 @@ func xorBitmapRun(a, b *Container) *Container {
return output
}
// CompareEquality is used mostly in test cases to confirm that two bitmaps came
// CompareBitmapSlice checks whether a bitmap has the same values in it
// that a provided slice does.
func CompareBitmapSlice(b *Bitmap, vals []uint64) (bool, error) {
count := b.Count()
if count != uint64(len(vals)) {
return false, fmt.Errorf("length mismatch: bitmap has %d bits, slice has %d", count, len(vals))
}
for _, v := range vals {
if !b.Contains(v) {
return false, fmt.Errorf("bitmap lacks expected value %d", v)
}
}
return true, nil
}
// CompareBitmapMap checks whether a bitmap has the same values in it
// that a provided map[uint64]struct{} has as keys.
func CompareBitmapMap(b *Bitmap, vals map[uint64]struct{}) (bool, error) {
count := b.Count()
if count != uint64(len(vals)) {
return false, fmt.Errorf("length mismatch: bitmap has %d bits, map has %d", count, len(vals))
}
for v := range vals {
if !b.Contains(v) {
return false, fmt.Errorf("bitmap lacks expected value %d", v)
}
}
return true, nil
}
// BitwiseEqual is used mostly in test cases to confirm that two bitmaps came
// out the same. It does not expect corresponding opN, or OpWriter, but expects
// identical bit contents. It does not expect identical representations; a bitmap
// container can be identical to an array container. It returns a boolean value,
@ -5825,35 +5900,6 @@ func (b *Bitmap) BitwiseEqual(c *Bitmap) (bool, error) {
return true, nil
}
func bitmapsEqual(b, c *Bitmap) error { // nolint: deadcode
statsHit("bitmapsEqual")
if b.OpWriter != c.OpWriter {
return errors.New("opWriters not equal")
}
if b.opN != c.opN {
return errors.New("opNs not equal")
}
biter, _ := b.Containers.Iterator(0)
citer, _ := c.Containers.Iterator(0)
bn, cn := biter.Next(), citer.Next()
for ; bn && cn; bn, cn = biter.Next(), citer.Next() {
bk, bc := biter.Value()
ck, cc := citer.Value()
if bk != ck {
return errors.New("keys not equal")
}
if !bc.equals(cc) {
return errors.New("containers not equal")
}
}
if bn && !cn || cn && !bn {
return errors.New("different numbers of containers")
}
return nil
}
func popcount(x uint64) uint64 {
return uint64(bits.OnesCount64(x))
}

89
roaring/roaring_helpers_test.go
View file

@ -14,6 +14,8 @@
package roaring
import "sync"
///////////////////////////////////////////////////////////////////////////
var containerWidth uint64 = 65536
@ -261,51 +263,56 @@ func doContainer(typ byte, data interface{}) *Container {
return nil
}
var makeCts sync.Once
var sampleTestContainers map[byte]map[string]*Container
func setupContainerTests() map[byte]map[string]*Container {
cts := make(map[byte]map[string]*Container)
makeCts.Do(func() {
sampleTestContainers = make(map[byte]map[string]*Container)
// array containers
cts[containerArray] = map[string]*Container{
"empty": doContainer(containerArray, arrayEmpty()),
"full": doContainer(containerArray, arrayFull()),
"firstBitSet": doContainer(containerArray, arrayFirstBitSet()),
"lastBitSet": doContainer(containerArray, arrayLastBitSet()),
"firstBitUnset": doContainer(containerArray, arrayFirstBitUnset()),
"lastBitUnset": doContainer(containerArray, arrayLastBitUnset()),
"innerBitsSet": doContainer(containerArray, arrayInnerBitsSet()),
"outerBitsSet": doContainer(containerArray, arrayOuterBitsSet()),
"oddBitsSet": doContainer(containerArray, arrayOddBitsSet()),
"evenBitsSet": doContainer(containerArray, arrayEvenBitsSet()),
}
// array containers
sampleTestContainers[containerArray] = map[string]*Container{
"empty": doContainer(containerArray, arrayEmpty()),
"full": doContainer(containerArray, arrayFull()),
"firstBitSet": doContainer(containerArray, arrayFirstBitSet()),
"lastBitSet": doContainer(containerArray, arrayLastBitSet()),
"firstBitUnset": doContainer(containerArray, arrayFirstBitUnset()),
"lastBitUnset": doContainer(containerArray, arrayLastBitUnset()),
"innerBitsSet": doContainer(containerArray, arrayInnerBitsSet()),
"outerBitsSet": doContainer(containerArray, arrayOuterBitsSet()),
"oddBitsSet": doContainer(containerArray, arrayOddBitsSet()),
"evenBitsSet": doContainer(containerArray, arrayEvenBitsSet()),
}
// bitmap containers
cts[containerBitmap] = map[string]*Container{
"empty": doContainer(containerBitmap, bitmapEmpty()),
"full": doContainer(containerBitmap, bitmapFull()),
"firstBitSet": doContainer(containerBitmap, bitmapFirstBitSet()),
"lastBitSet": doContainer(containerBitmap, bitmapLastBitSet()),
"firstBitUnset": doContainer(containerBitmap, bitmapFirstBitUnset()),
"lastBitUnset": doContainer(containerBitmap, bitmapLastBitUnset()),
"innerBitsSet": doContainer(containerBitmap, bitmapInnerBitsSet()),
"outerBitsSet": doContainer(containerBitmap, bitmapOuterBitsSet()),
"oddBitsSet": doContainer(containerBitmap, bitmapOddBitsSet()),
"evenBitsSet": doContainer(containerBitmap, bitmapEvenBitsSet()),
}
// bitmap containers
sampleTestContainers[containerBitmap] = map[string]*Container{
"empty": doContainer(containerBitmap, bitmapEmpty()),
"full": doContainer(containerBitmap, bitmapFull()),
"firstBitSet": doContainer(containerBitmap, bitmapFirstBitSet()),
"lastBitSet": doContainer(containerBitmap, bitmapLastBitSet()),
"firstBitUnset": doContainer(containerBitmap, bitmapFirstBitUnset()),
"lastBitUnset": doContainer(containerBitmap, bitmapLastBitUnset()),
"innerBitsSet": doContainer(containerBitmap, bitmapInnerBitsSet()),
"outerBitsSet": doContainer(containerBitmap, bitmapOuterBitsSet()),
"oddBitsSet": doContainer(containerBitmap, bitmapOddBitsSet()),
"evenBitsSet": doContainer(containerBitmap, bitmapEvenBitsSet()),
}
// run containers
cts[containerRun] = map[string]*Container{
"empty": doContainer(containerRun, runEmpty()),
"full": doContainer(containerRun, runFull()),
"firstBitSet": doContainer(containerRun, runFirstBitSet()),
"lastBitSet": doContainer(containerRun, runLastBitSet()),
"firstBitUnset": doContainer(containerRun, runFirstBitUnset()),
"lastBitUnset": doContainer(containerRun, runLastBitUnset()),
"innerBitsSet": doContainer(containerRun, runInnerBitsSet()),
"outerBitsSet": doContainer(containerRun, runOuterBitsSet()),
"oddBitsSet": doContainer(containerRun, runOddBitsSet()),
"evenBitsSet": doContainer(containerRun, runEvenBitsSet()),
}
// run containers
sampleTestContainers[containerRun] = map[string]*Container{
"empty": doContainer(containerRun, runEmpty()),
"full": doContainer(containerRun, runFull()),
"firstBitSet": doContainer(containerRun, runFirstBitSet()),
"lastBitSet": doContainer(containerRun, runLastBitSet()),
"firstBitUnset": doContainer(containerRun, runFirstBitUnset()),
"lastBitUnset": doContainer(containerRun, runLastBitUnset()),
"innerBitsSet": doContainer(containerRun, runInnerBitsSet()),
"outerBitsSet": doContainer(containerRun, runOuterBitsSet()),
"oddBitsSet": doContainer(containerRun, runOddBitsSet()),
"evenBitsSet": doContainer(containerRun, runEvenBitsSet()),
}
})
return cts
return sampleTestContainers
}

103
roaring/roaring_internal_test.go
View file

@ -2632,7 +2632,7 @@ func TestBitmapClone(t *testing.T) {
}
}
c := b.Clone()
if err := bitmapsEqual(b, c); err != nil {
if _, err := b.BitwiseEqual(c); err != nil {
t.Fatalf("Clone Objects not equal: %v\n", err)
}
d := func() *Bitmap { //anybody know how to declare a nil value?
@ -2735,34 +2735,46 @@ func getFunctionName(i interface{}) string {
return y[0]
}
// UnionInPlace is defined at the Bitmap level, but this wrapper lets us insert
// it into our ContainerCombinations tests so that it gets exercised on a wide
// variety of container data.
func unionInPlaceWrapper(a, b *Container) *Container {
out := NewBitmap()
out.Containers.Put(0, a.Clone())
B := NewBitmap()
B.Containers.Put(0, b)
out.UnionInPlace(B)
return out.Containers.Get(0)
ret := a.Clone().unionInPlace(b)
ret.Repair()
return ret
}
func differenceInPlaceWrapper(a, b *Container) *Container {
out := NewBitmap()
out.Containers.Put(0, a.Clone())
B := NewBitmap()
B.Containers.Put(0, b)
out.DifferenceInPlace(B)
return out.Containers.Get(0)
a = a.Clone()
// this should probably return its new value, but currently does not
a.differenceInPlace(b)
return a
}
func intersectInPlaceWrapper(a, b *Container) *Container {
out := NewBitmap()
out.Containers.Put(0, a.Clone())
B := NewBitmap()
B.Containers.Put(0, b)
out.IntersectInPlace(B)
return out.Containers.Get(0)
return a.Clone().intersectInPlace(b)
}
func TestContainerBitwiseCompare(t *testing.T) {
cts := setupContainerTests()
for t1, containers := range cts {
for name, c := range containers {
for t2, other := range cts {
for otherName, otherC := range other {
err := c.BitwiseCompare(otherC)
if err != nil {
if otherName == name {
t.Fatalf("container types %d/%d, contents %s: unexpected error %v",
t1, t2, name, err)
}
} else {
if name != otherName {
t.Fatalf("container types %d/%d, unexpected %s == %s",
t1, t2, name, otherName)
}
}
}
}
}
}
}
func TestContainerCombinations(t *testing.T) {
@ -3562,51 +3574,8 @@ func TestContainerCombinations(t *testing.T) {
// Convert to all container types and check result.
for _, ct := range containerTypes {
clone := ret.Clone()
if ct == containerArray {
if clone == nil {
clone = NewContainerArray(nil)
} else if clone.isBitmap() {
clone = clone.bitmapToArray()
} else if clone.isRun() {
clone = clone.runToArray()
}
if clone.N() != cts[ct][exp].N() {
t.Errorf("test %s expected array n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N())
}
// Because xorRunRun resulting in an empty container returns an array container with a
// nil slice array, then we need to check len() on array first (look for 0).
if !(len(clone.array()) == 0 && len(cts[ct][exp].array()) == 0) && !reflect.DeepEqual(clone.array(), cts[ct][exp].array()) {
t.Errorf("test %s expected array %X, but got %X", desc, cts[ct][exp].array(), clone.array())
}
} else if ct == containerBitmap {
if clone == nil {
clone = NewContainerBitmap(0, nil)
} else if clone.isArray() {
clone = clone.arrayToBitmap()
} else if clone.isRun() {
clone = clone.runToBitmap()
}
if clone.N() != cts[ct][exp].N() {
t.Errorf("test %s expected bitmap n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N())
}
if !reflect.DeepEqual(clone.bitmap(), cts[ct][exp].bitmap()) {
t.Errorf("test %s expected bitmap %X, but got %X", desc, cts[ct][exp].bitmap(), clone.bitmap())
}
} else if ct == containerRun {
if clone == nil {
clone = NewContainerRun(nil)
} else if clone.isArray() {
clone = clone.arrayToRun(0)
} else if clone.isBitmap() {
clone = clone.bitmapToRun(0)
}
if clone.N() != cts[ct][exp].N() {
t.Errorf("test %s expected runs n=%d, but got n=%d", desc, cts[ct][exp].N(), clone.N())
}
if !reflect.DeepEqual(clone.runs(), cts[ct][exp].runs()) {
t.Errorf("test %s expected runs %X, but got %X", desc, cts[ct][exp].runs(), clone.runs())
}
if err := ret.BitwiseCompare(cts[ct][exp]); err != nil {
t.Errorf("test %s: %v", desc, err)
}
}
}

40
roaring/roaring_test.go
View file

@ -866,7 +866,7 @@ func TestBitmap_UnionInPlaceProp(t *testing.T) {
seed = time.Now().UnixNano()
source = rand.NewSource(seed)
rng = rand.New(source)
numTests = 100
numTests = 20
maxNumIntsPerBatch = 100
maxNumBatches = 100
maxRangePercent = 2
@ -1425,10 +1425,10 @@ func TestBitmap_Shift(t *testing.T) {
}
func TestBitmap_Quick_Array1(t *testing.T) { testBitmapQuick(t, 1000, 1000, 2000) }
func TestBitmap_Quick_Array2(t *testing.T) { testBitmapQuick(t, 10000, 0, 1000) }
func TestBitmap_Quick_Bitmap1(t *testing.T) { testBitmapQuick(t, 10000, 0, 10000) }
func TestBitmap_Quick_Bitmap2(t *testing.T) { testBitmapQuick(t, 10000, 10000, 20000) }
func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 10000, 0, math.MaxInt64) }
func TestBitmap_Quick_Array2(t *testing.T) { testBitmapQuick(t, 1000, 0, 1000) }
func TestBitmap_Quick_Bitmap1(t *testing.T) { testBitmapQuick(t, 1000, 0, 10000) }
func TestBitmap_Quick_Bitmap2(t *testing.T) { testBitmapQuick(t, 1000, 10000, 20000) }
func TestBitmap_Quick_LargeValue(t *testing.T) { testBitmapQuick(t, 1000, 0, math.MaxInt64) }
// Ensure a bitmap can perform basic operations on randomly generated values.
func testBitmapQuick(t *testing.T, n int, min, max uint64) {
@ -1504,20 +1504,20 @@ func TestBitmap_Marshal_Quick_Array1(t *testing.T) {
testBitmapMarshalQuick(t, 1000, 1000, 2000, false)
}
func TestBitmap_Marshal_Quick_Array2(t *testing.T) {
testBitmapMarshalQuick(t, 10000, 0, 1000, false)
testBitmapMarshalQuick(t, 1000, 0, 1000, false)
}
func TestBitmap_Marshal_Quick_Bitmap1(t *testing.T) {
testBitmapMarshalQuick(t, 10000, 0, 10000, false)
testBitmapMarshalQuick(t, 1000, 0, 10000, false)
}
func TestBitmap_Marshal_Quick_Bitmap2(t *testing.T) {
testBitmapMarshalQuick(t, 10000, 10000, 20000, false)
testBitmapMarshalQuick(t, 1000, 10000, 20000, false)
}
func TestBitmap_Marshal_Quick_LargeValue(t *testing.T) {
testBitmapMarshalQuick(t, 100, 0, math.MaxInt64, false)
}
func TestBitmap_Marshal_Quick_Bitmap_Sorted(t *testing.T) {
testBitmapMarshalQuick(t, 10000, 0, 10000, true)
testBitmapMarshalQuick(t, 1000, 0, 10000, true)
}
// TODO update for RLE
@ -1570,14 +1570,12 @@ func testBitmapMarshalQuick(t *testing.T, n int, min, max uint64, sorted bool) {
t.Fatal(err)
}
// Verify the original bitmap has the correct set of values.
if exp, got := generator.Uint64SetSlice(set), bm.Slice(); !reflect.DeepEqual(exp, got) {
t.Fatalf("mismatch: %s\n\nexp=%+v\n\ngot=%+v\n\n", diff(exp, got), exp, got)
if _, err := roaring.CompareBitmapMap(bm, set); err != nil {
t.Fatalf("source mismatch: %v", err)
}
// Verify the bitmap loaded with the ops log has the correct set of values.
if exp, got := generator.Uint64SetSlice(set), bm2.Slice(); !reflect.DeepEqual(exp, got) {
t.Fatalf("mismatch: %s\n\nexp=%+v\n\ngot=%+v\n\n", diff(exp, got), exp, got)
if _, err := roaring.CompareBitmapMap(bm2, set); err != nil {
t.Fatalf("unmarshalled mismatch: %v", err)
}
}
@ -1839,18 +1837,6 @@ func getBenchData(tb testing.TB) *benchmarkSampleData {
return data
}
func diff(a, b []uint64) string {
if len(a) != len(b) {
return fmt.Sprintf("len: %d != %d", len(a), len(b))
}
for i := range a {
if a[i] != b[i] {
return fmt.Sprintf("index %d: %d != %d", i, a[i], b[i])
}
}
return ""
}
func TestBitmap_Intersect(t *testing.T) {
bm0 := testBM()
result := bm0.Intersect(bm0)

18
server.go
View file

@ -77,6 +77,8 @@ type Server struct { // nolint: maligned
metricInterval time.Duration
diagnosticInterval time.Duration
maxWritesPerRequest int
confirmDownSleep time.Duration
confirmDownRetries int
isCoordinator bool
syncer holderSyncer
@ -229,6 +231,17 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption {
}
}
// OptServerNodeDownRetries is a functional option on Server
// used to specify the retries and sleep duration for node down
// checks.
func OptServerNodeDownRetries(retries int, sleep time.Duration) ServerOption {
return func(s *Server) error {
s.confirmDownRetries = retries
s.confirmDownSleep = sleep
return nil
}
}
// OptServerURI is a functional option on Server
// used to set the server URI.
func OptServerURI(uri *URI) ServerOption {
@ -330,6 +343,9 @@ func NewServer(opts ...ServerOption) (*Server, error) {
metricInterval: 0,
diagnosticInterval: 0,
confirmDownRetries: defaultConfirmDownRetries,
confirmDownSleep: defaultConfirmDownSleep,
resetTranslationSyncCh: make(chan struct{}),
logger: logger.NopLogger,
@ -399,6 +415,8 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.executor.MaxWritesPerRequest = s.maxWritesPerRequest
s.cluster.broadcaster = s
s.cluster.maxWritesPerRequest = s.maxWritesPerRequest
s.cluster.confirmDownRetries = s.confirmDownRetries
s.cluster.confirmDownSleep = s.confirmDownSleep
s.holder.broadcaster = s
err = s.loadAllExtensions()
if err != nil {

14
test/pilosa.go
View file

@ -92,7 +92,7 @@ func NewCommandNode(isCoordinator bool, opts ...server.CommandOption) *Command {
// We want tests to default to using the in-memory translate store, so we
// prepend opts with that functional option. If a different translate store
// has been specified, it will override this one.
opts = prependWithMemStore(opts)
opts = prependTestServerOpts(opts)
m := newCommand(opts...)
m.Config.Cluster.Disabled = false
m.Config.Cluster.Coordinator = isCoordinator
@ -434,25 +434,25 @@ func MustRunCluster(tb testing.TB, size int, opts ...[]server.CommandOption) Clu
return c
}
// prependOpts applies prependWithMemStore to each of the ops (one per
// prependOpts applies prependTestServerOpts to each of the ops (one per
// node, or one for the entire cluser).
func prependOpts(opts [][]server.CommandOption) [][]server.CommandOption {
if len(opts) == 0 {
opts = [][]server.CommandOption{
prependWithMemStore([]server.CommandOption{}),
prependTestServerOpts([]server.CommandOption{}),
}
} else {
for i := range opts {
opts[i] = prependWithMemStore(opts[i])
opts[i] = prependTestServerOpts(opts[i])
}
}
return opts
}
// prependWithMemStore prepends opts with the OpenInMemTranslateStore.
func prependWithMemStore(opts []server.CommandOption) []server.CommandOption {
// prependTestServerOpts prepends opts with the OpenInMemTranslateStore.
func prependTestServerOpts(opts []server.CommandOption) []server.CommandOption {
defaultOpts := []server.CommandOption{
server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore)),
server.OptCommandServerOptions(pilosa.OptServerOpenTranslateStore(pilosa.OpenInMemTranslateStore), pilosa.OptServerNodeDownRetries(5, 100*time.Millisecond)),
}
return append(defaultOpts, opts...)
}