mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
ad30a926f4
These commits are hard to disentagle, and doing them separately means re-modifying the same chunks of code several times before removing it, and similar things. Basically: (1) Drop the bolt backend storage. (2) Drop the blue-green wrapper that compares two backends. (3) Drop unused or barely-used Tx API components from all the remaining backends. (4) Minor related cleanup to simplify things related to these. The boltdb backend existed only to verify RBF. The blue-green wrapper was mostly used to verify RBF, but in practice we had to do a lot of working around that, and it introduced a lot of special cases. Types removed: IteratorFinder: Used only to implement the roaring iterator on top of boltdb, and to complicate the way it worked in roaring. Reverted the complications. Also unexport NewSliceContainers which is used only for that outside of roaring's internals. PortMapper from cluster_internal_test.go: Used only for a test we removed early this year. Never used for anything else. RawRoaringData: Totally unused. TxStore: Totally unused. Functions removed from Tx API, and sometimes corresponding members were removed from structs: * Dump: debugging code, I don't think I found any actually reachable paths to it. * Group: only used for debugging TxGroup stuff * IncrementOpN: only used by fragment, fragment can increment its own opN. * Options: unused? * Pointer: debugging only * Readonly: used only to decide how to handle Tx in a TxGrp, but we never add a non-readonly Tx to a TxGrp. Removed also all the corresponding write-aware stuff. * RoaringBitmapReader: Used exactly once, can just be a bm.WriteTo. * Sn (and OpenSnList): Unused * UnionInPlace: unused and conceptually-invalid; it didn't write to storage and shouldn't have, and was just "create a bitmap then call union-in-place", which we can do directly. * UseRowCache: just checked storage.UseRowCache. Other things removed: The SetRequiredForAtomicWriteTx and ClearRequiredForAtomicWriteTx functions go away, since nothing now seems to be using them? Same for holder_internal_test's `testHasBit` and `testMustNotHaveBit`, which were unused. The DBPerShard "DeleteDBPath" and "HasData" functions and related parts were mostly unused; took out the parts that were never actually being reached. Changed the API of one function to simplify special cases and remove things: * ImportRoaringBits had a special "data" argument which gave it subtly different semantics for RBF and roaring (for roaring, it could produce a roaring bitmap *with ops log*), didn't seem to be adding much. Removed corresponding "readStorageFromArchive" which is not otherwise used. Also took out various debugging/dumping functions that were unused and may have bitrotted. Dropped a test from txfactory_internal_test, and the "pjobs" code, because those two were the only things that needed Barrier and thus idem, which lets us drop two more dependencies. We already have errgroup for grouping things which want to terminate as soon as one of them errors, approximately. To do better we'd have to have context-threading, really. Unbroke the WriteFragment test for non-roaring tests and made it not roaring-only.
121 lines
3 KiB
Go
121 lines
3 KiB
Go
// Copyright 2017 Pilosa Corp.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package pilosa
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
pnet "github.com/molecula/featurebase/v2/net"
|
|
"github.com/molecula/featurebase/v2/testhook"
|
|
"github.com/molecula/featurebase/v2/topology"
|
|
)
|
|
|
|
// utilities used by tests
|
|
|
|
// NewTestCluster returns a cluster with n nodes and uses a mod-based hasher.
|
|
func NewTestCluster(tb testing.TB, n int) *cluster {
|
|
path, err := testhook.TempDir(tb, "pilosa-cluster-")
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
availableShardFileFlushDuration.Set(100 * time.Millisecond)
|
|
c := newCluster()
|
|
c.ReplicaN = 1
|
|
c.Hasher = NewTestModHasher()
|
|
c.Path = path
|
|
|
|
for i := 0; i < n; i++ {
|
|
c.noder.AppendNode(&topology.Node{
|
|
ID: fmt.Sprintf("node%d", i),
|
|
URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)),
|
|
})
|
|
}
|
|
|
|
cNodes := c.noder.Nodes()
|
|
|
|
c.Node = cNodes[0]
|
|
return c
|
|
}
|
|
|
|
// NewTestURI is a test URI creator that intentionally swallows errors.
|
|
func NewTestURI(scheme, host string, port uint16) pnet.URI {
|
|
uri := pnet.DefaultURI()
|
|
_ = uri.SetScheme(scheme)
|
|
_ = uri.SetHost(host)
|
|
uri.SetPort(port)
|
|
return *uri
|
|
}
|
|
|
|
func NewTestURIFromHostPort(host string, port uint16) pnet.URI {
|
|
uri := pnet.DefaultURI()
|
|
_ = uri.SetHost(host)
|
|
uri.SetPort(port)
|
|
return *uri
|
|
}
|
|
|
|
// ModHasher represents a simple, mod-based hashing.
|
|
type TestModHasher struct{}
|
|
|
|
// NewTestModHasher returns a new instance of ModHasher with n buckets.
|
|
func NewTestModHasher() *TestModHasher { return &TestModHasher{} }
|
|
|
|
func (*TestModHasher) Hash(key uint64, n int) int { return int(key) % n }
|
|
|
|
func (*TestModHasher) Name() string { return "mod" }
|
|
|
|
var _ = NewTestClusterWithReplication // happy linter
|
|
|
|
func NewTestClusterWithReplication(tb testing.TB, nNodes, nReplicas, partitionN int) (c *cluster, cleaner func()) {
|
|
path, err := testhook.TempDir(tb, "pilosa-cluster-")
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
// holder
|
|
h := NewHolder(path, mustHolderConfig())
|
|
|
|
// cluster
|
|
availableShardFileFlushDuration.Set(100 * time.Millisecond)
|
|
c = newCluster()
|
|
c.holder = h
|
|
c.ReplicaN = nReplicas
|
|
c.Hasher = &topology.Jmphasher{}
|
|
c.Path = path
|
|
c.partitionN = partitionN
|
|
|
|
for i := 0; i < nNodes; i++ {
|
|
nodeID := fmt.Sprintf("node%d", i)
|
|
c.noder.AppendNode(&topology.Node{
|
|
ID: nodeID,
|
|
URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)),
|
|
})
|
|
}
|
|
|
|
cNodes := c.noder.Nodes()
|
|
|
|
c.Node = cNodes[0]
|
|
|
|
if err := c.holder.Open(); err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
return c, func() {
|
|
c.holder.Close()
|
|
c.close()
|
|
}
|
|
}
|