scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

address some test failures due to random ordered etcd ID

Browse source
This commit is contained in:
Travis 2021-01-25 00:52:49 -06:00
parent 2f66501160
commit ace4dea46f
No known key found for this signature in database
GPG key ID: 37080CC2042BA34E
12 changed files with 135 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
cluster.go
View file

@ -74,7 +74,8 @@ type nodeAction struct {
// cluster represents a collection of nodes.
type cluster struct { // nolint: maligned
noder topology.Noder
noder topology.Noder
unprotectedNoder topology.Noder
id string
Node *topology.Node
@ -161,10 +162,41 @@ func newCluster() *cluster {
confirmDownRetries: defaultConfirmDownRetries,
confirmDownSleep: defaultConfirmDownSleep,
}
c.noder = c // TODO: this is temporary until etcd fully implements noder
// TODO: these are temporary until etcd fully implements noder
c.noder = c
c.unprotectedNoder = &unprotectedCluster{
c: c,
}
return c
}
// unprotectedCluster is a temporary struct used in cases of NewClusterSnapshot
// which are inside of a c.mu.Lock(). These cases can't use the normal c.noder
// (which is also temporary), because c.Nodes() aquires c.mu.Lock() as well.
type unprotectedCluster struct {
c *cluster
}
// Nodes returns a copy of the slice of nodes in the cluster.
func (uc *unprotectedCluster) Nodes() []*topology.Node {
ret := make([]*topology.Node, len(uc.c.nodes))
copy(ret, uc.c.nodes)
return ret
}
// SetNodes implements the Noder interface.
func (uc *unprotectedCluster) SetNodes(nodes []*topology.Node) {}
// AppendNode implements the Noder interface.
func (uc *unprotectedCluster) AppendNode(node *topology.Node) {}
// RemoveNode implements the Noder interface.
func (uc *unprotectedCluster) RemoveNode(nodeID string) bool {
return false
}
// initializeAntiEntropy is called by the anti entropy routine when it starts.
// If the AE channel is created without a routine reading from it, cluster will
// block indefinitely when calling abortAntiEntropy().
@ -667,7 +699,7 @@ func (c *cluster) fragsByHost(idx *Index) fragsByHost {
// for the given set of shards with data.
func (c *cluster) fragCombos(idx string, availableShards *roaring.Bitmap, fieldViews viewsByField) fragsByHost {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(c.noder, c.Hasher, c.ReplicaN)
snap := topology.NewClusterSnapshot(c.unprotectedNoder, c.Hasher, c.ReplicaN)
t := make(fragsByHost)
_ = availableShards.ForEach(func(i uint64) error {
@ -832,8 +864,8 @@ func (c *cluster) translationNodes(to *cluster) (map[string][]*translationResize
}
// Create a snapshot of the cluster to use for node/partition calculations.
fSnap := topology.NewClusterSnapshot(c.noder, c.Hasher, c.ReplicaN)
toSnap := topology.NewClusterSnapshot(to.noder, c.Hasher, to.ReplicaN)
fSnap := topology.NewClusterSnapshot(c.unprotectedNoder, c.Hasher, c.ReplicaN)
toSnap := topology.NewClusterSnapshot(to.unprotectedNoder, c.Hasher, to.ReplicaN)
for pid := 0; pid < c.partitionN; pid++ {
fNodes := fSnap.PartitionNodes(pid)
@ -1466,7 +1498,7 @@ func (c *cluster) unprotectedGenerateResizeJobByAction(nodeAction nodeAction) (*
}
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(c.noder, c.Hasher, c.ReplicaN)
snap := topology.NewClusterSnapshot(c.unprotectedNoder, c.Hasher, c.ReplicaN)
instr := &ResizeInstruction{
JobID: j.ID,
@ -1493,7 +1525,7 @@ func (c *cluster) completeCurrentJob(state string) error {
func (c *cluster) unprotectedCompleteCurrentJob(state string) error {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(c.noder, c.Hasher, c.ReplicaN)
snap := topology.NewClusterSnapshot(c.unprotectedNoder, c.Hasher, c.ReplicaN)
// TODO: this needs to become: IsPrimaryFieldTranslationNode(c.Node.ID)
if !snap.IsCoordinatorNode(c.Node.ID) {
return ErrNodeNotCoordinator
@ -1657,7 +1689,6 @@ func (c *cluster) followResizeInstruction(instr *ResizeInstruction) error {
}
func (c *cluster) markResizeInstructionComplete(complete *ResizeInstructionComplete) error {
j := c.job(complete.JobID)
// Abort the job if an error exists in the complete object.
@ -2454,7 +2485,7 @@ func (c *cluster) findFieldKeys(ctx context.Context, field *Field, keys ...strin
}
if !field.Keys() {
return nil, errors.Wrap(ErrTranslatingKeyNotFound, "field is not keyed")
return nil, errors.Wrap(ErrTranslatingKeyNotFound, "field is not keyed 1")
}
// Attempt to find the keys locally.
@ -2517,7 +2548,7 @@ func (c *cluster) createFieldKeys(ctx context.Context, field *Field, keys ...str
}
if !field.Keys() {
return nil, errors.Wrap(ErrTranslatingKeyNotFound, "field is not keyed")
return nil, errors.Wrap(ErrTranslatingKeyNotFound, "field is not keyed 2")
}
// The coordinator is the only node that can create field keys, since it owns the authoritative copy.

12
cluster_internal_test.go
View file

@ -892,6 +892,13 @@ func TestCluster_ResizeStates(t *testing.T) {
t.Fatal(err)
}
// Close TestCluster with defer.
defer func() {
if err := tc.Close(); err != nil {
t.Fatal(err)
}
}()
// Add Bit Data to node0.
if err := tc.CreateField("i", "f", OptFieldTypeDefault()); err != nil {
t.Fatalf("creating field: %v", err)
@ -962,11 +969,6 @@ func TestCluster_ResizeStates(t *testing.T) {
} else if !bytes.Equal(chksum, node0Checksum) {
t.Fatalf("expected standard view checksum to match: %x - %x", chksum, node0Checksum)
}
// Close TestCluster.
if err := tc.Close(); err != nil {
t.Fatal(err)
}
})
}

2
cmd/pilosa-fsck/fsck_test.go
View file

@ -32,7 +32,7 @@ import (
)
func Test_Repair(t *testing.T) {
t.Skip("I don't quite understand what this test is doing and will need help adjusting it to pass again.")
// a) setup 1 primary + 3 replicas of disagree-ing cluster dirs.
nNodes := 4

2
holder.go
View file

@ -1838,7 +1838,7 @@ func (c *holderCleaner) IsClosing() bool {
// any unnecessary fragments and files.
func (c *holderCleaner) CleanHolder() error {
// Create a snapshot of the cluster to use for node/partition calculations.
snap := topology.NewClusterSnapshot(c.Cluster.noder, c.Cluster.Hasher, c.Cluster.ReplicaN)
snap := topology.NewClusterSnapshot(c.Cluster.unprotectedNoder, c.Cluster.Hasher, c.Cluster.ReplicaN)
for _, index := range c.Holder.Indexes() {
// Verify cleaner has not closed.

2
holder_test.go
View file

@ -605,6 +605,8 @@ func TestHolderSyncer_Clears(t *testing.T) {
c.GetIdleNode(0).Config.AntiEntropy.Interval = 0
c.GetIdleNode(1).Config.Cluster.ReplicaN = 3
c.GetIdleNode(1).Config.AntiEntropy.Interval = 0
c.GetIdleNode(2).Config.Cluster.ReplicaN = 3
c.GetIdleNode(2).Config.AntiEntropy.Interval = 0
err := c.Start()
if err != nil {
t.Fatalf("starting cluster: %v", err)

18
http/client.go
View file

@ -884,7 +884,7 @@ func (c *InternalClient) CreateField(ctx context.Context, index, field string) e
return c.CreateFieldWithOptions(ctx, index, field, pilosa.FieldOptions{})
}
// CreateField creates a new field on the server.
// CreateFieldWithOptions creates a new field on the server.
func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, field string, opt pilosa.FieldOptions) error {
span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.CreateFieldWithOptions")
defer span.Finish()
@ -902,20 +902,26 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
// should probably happen in the field anyway??
fieldOpt := fieldOptions{
Type: opt.Type,
Keys: &opt.Keys,
}
if fieldOpt.Type == pilosa.FieldTypeSet {
switch fieldOpt.Type {
case pilosa.FieldTypeSet, pilosa.FieldTypeMutex:
fieldOpt.CacheType = &opt.CacheType
fieldOpt.CacheSize = &opt.CacheSize
} else if fieldOpt.Type == pilosa.FieldTypeInt {
fieldOpt.Keys = &opt.Keys
case pilosa.FieldTypeInt:
fieldOpt.Min = &opt.Min
fieldOpt.Max = &opt.Max
} else if fieldOpt.Type == pilosa.FieldTypeTime {
case pilosa.FieldTypeTime:
fieldOpt.TimeQuantum = &opt.TimeQuantum
} else if fieldOpt.Type == pilosa.FieldTypeDecimal {
case pilosa.FieldTypeBool:
// pass
case pilosa.FieldTypeDecimal:
fieldOpt.Min = &opt.Min
fieldOpt.Max = &opt.Max
fieldOpt.Scale = &opt.Scale
default:
fieldOpt.Type = pilosa.DefaultFieldType
fieldOpt.Keys = &opt.Keys
}
// TODO: remove buf completely? (depends on whether importer needs to create specific field types)

19
http/client_test.go
View file

@ -1225,8 +1225,23 @@ func TestClientTransactions(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
client0 := MustNewClient(c.GetNode(0).URL(), http.GetHTTPClient(nil))
client1 := MustNewClient(c.GetNode(1).URL(), http.GetHTTPClient(nil))
coord := c.GetCoordinator()
if coord == nil {
t.Fatal("no coordinator node")
}
var other *test.Command
node0 := c.GetNode(0)
node1 := c.GetNode(1)
if coord == node0 {
other = node1
} else {
other = node0
}
client0 := MustNewClient(coord.URL(), http.GetHTTPClient(nil))
client1 := MustNewClient(other.URL(), http.GetHTTPClient(nil))
// can create, list, get, and finish a transaction
var expDeadline time.Time

39
test/cluster.go
View file

@ -57,7 +57,7 @@ func (c *Cluster) Query(t testing.TB, index, query string) pilosa.QueryResponse
t.Fatal("must have at least one node in cluster to query")
}
return c.Nodes[0].QueryAPI(t, &pilosa.QueryRequest{Index: index, Query: query})
return c.GetNode(0).QueryAPI(t, &pilosa.QueryRequest{Index: index, Query: query})
}
// QueryHTTP executes a PQL query through the HTTP endpoint. It fails
@ -69,7 +69,7 @@ func (c *Cluster) QueryHTTP(t testing.TB, index, query string) (string, error) {
t.Fatal("must have at least one node in cluster to query")
}
return c.Nodes[0].Query(t, index, "", query)
return c.GetNode(0).Query(t, index, "", query)
}
// QueryGRPC executes a PQL query through the GRPC endpoint. It fails the
@ -80,7 +80,7 @@ func (c *Cluster) QueryGRPC(t testing.TB, index, query string) *proto.TableRespo
t.Fatal("must have at least one node in cluster to query")
}
grpcClient, err := client.NewGRPCClient([]string{fmt.Sprintf("%s:%d", c.Nodes[0].Server.GRPCURI().Host, c.Nodes[0].Server.GRPCURI().Port)}, nil)
grpcClient, err := client.NewGRPCClient([]string{fmt.Sprintf("%s:%d", c.GetNode(0).Server.GRPCURI().Host, c.GetNode(0).Server.GRPCURI().Port)}, nil)
if err != nil {
t.Fatalf("getting GRPC client: %v", err)
}
@ -134,6 +134,19 @@ func (c *Cluster) GetNode(n int) *Command {
return c.Nodes[ids[n].idx]
}
// GetCoordinator gets the node which has been determined to be the coordinator.
// This used to be node0 in tests, but since implementing etcd, the coordinator
// can be any node in the cluster, so we have to use this method in tests which
// need to act on the coordinator.
func (c *Cluster) GetCoordinator() *Command {
for i := range c.Nodes {
if c.Nodes[i].IsCoordinator() {
return c.Nodes[i]
}
}
return nil
}
// nodePlace represents a node's ID and its index into the c.Nodes slice.
type nodePlace struct {
id string
@ -141,7 +154,7 @@ type nodePlace struct {
}
func (c *Cluster) GetHolder(n int) *Holder {
return &Holder{Holder: c.Nodes[n].Server.Holder()}
return &Holder{Holder: c.GetNode(n).Server.Holder()}
}
func (c *Cluster) Len() int {
@ -163,7 +176,7 @@ func (c *Cluster) ImportBits(t testing.TB, index, field string, rowcols [][2]uin
rowIDs[i] = bit[0]
colIDs[i] = bit[1]
}
nodes, err := c.Nodes[0].API.ShardNodes(context.Background(), index, shard)
nodes, err := c.GetNode(0).API.ShardNodes(context.Background(), index, shard)
if err != nil {
t.Fatalf("getting shard nodes: %v", err)
}
@ -206,7 +219,7 @@ func (c *Cluster) ImportKeyKey(t testing.TB, index, field string, valAndRecKeys
importRequest.RowKeys[i] = vk[0]
importRequest.ColumnKeys[i] = vk[1]
}
err := c.Nodes[0].API.Import(context.Background(), nil, importRequest)
err := c.GetNode(0).API.Import(context.Background(), nil, importRequest)
if err != nil {
t.Fatalf("importing keykey data: %v", err)
}
@ -236,7 +249,7 @@ func (c *Cluster) ImportTimeQuantumKey(t testing.TB, index, field string, entrie
importRequest.Timestamps[i] = entry.Ts
}
err := c.Nodes[0].API.Import(context.Background(), nil, importRequest)
err := c.GetNode(0).API.Import(context.Background(), nil, importRequest)
if err != nil {
t.Fatalf("importing keykey data: %v", err)
}
@ -262,7 +275,7 @@ func (c *Cluster) ImportIntKey(t testing.TB, index, field string, pairs []IntKey
importRequest.Values[i] = pair.Val
importRequest.ColumnKeys[i] = pair.Key
}
if err := c.Nodes[0].API.ImportValue(context.Background(), nil, importRequest); err != nil {
if err := c.GetNode(0).API.ImportValue(context.Background(), nil, importRequest); err != nil {
t.Fatalf("importing IntKey data: %v", err)
}
}
@ -286,7 +299,7 @@ func (c *Cluster) ImportIntID(t testing.TB, index, field string, pairs []IntID)
importRequest.Values[i] = pair.Val
importRequest.ColumnIDs[i] = pair.ID
}
if err := c.Nodes[0].API.ImportValue(context.Background(), nil, importRequest); err != nil {
if err := c.GetNode(0).API.ImportValue(context.Background(), nil, importRequest); err != nil {
t.Fatalf("importing IntID data: %v", err)
}
}
@ -311,7 +324,7 @@ func (c *Cluster) ImportIDKey(t testing.TB, index, field string, pairs []KeyID)
importRequest.RowIDs[i] = pair.ID
importRequest.ColumnKeys[i] = pair.Key
}
err := c.Nodes[0].API.Import(context.Background(), nil, importRequest)
err := c.GetNode(0).API.Import(context.Background(), nil, importRequest)
if err != nil {
t.Fatalf("importing IDKey data: %v", err)
}
@ -320,11 +333,11 @@ func (c *Cluster) ImportIDKey(t testing.TB, index, field string, pairs []KeyID)
// CreateField creates the index (if necessary) and field specified.
func (c *Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOptions, field string, fopts ...pilosa.FieldOption) *pilosa.Field {
t.Helper()
idx, err := c.Nodes[0].API.CreateIndex(context.Background(), index, iopts)
idx, err := c.GetNode(0).API.CreateIndex(context.Background(), index, iopts)
if err != nil && !strings.Contains(err.Error(), "index already exists") {
t.Fatalf("creating index: %v", err)
} else if err != nil { // index exists
idx, err = c.Nodes[0].API.Index(context.Background(), index)
idx, err = c.GetNode(0).API.Index(context.Background(), index)
if err != nil {
t.Fatalf("getting index: %v", err)
}
@ -333,7 +346,7 @@ func (c *Cluster) CreateField(t testing.TB, index string, iopts pilosa.IndexOpti
t.Logf("existing index options:\n%v\ndon't match given opts:\n%v\n in pilosa/test.Cluster.CreateField", idx.Options(), iopts)
}
f, err := c.Nodes[0].API.CreateField(context.Background(), index, field, fopts...)
f, err := c.GetNode(0).API.CreateField(context.Background(), index, field, fopts...)
// we'll assume the field doesn't exist because checking if the options
// match seems painful.
if err != nil {

3
test/pilosa.go
View file

@ -192,6 +192,9 @@ func (m *Command) URL() string { return m.API.Node().URI.String() }
// ID returns the node ID used by the running program.
func (m *Command) ID() string { return m.API.Node().ID }
// IsCoordinator returns true if this is the coordinator.
func (m *Command) IsCoordinator() bool { return m.API.Node().IsCoordinator }
// Client returns a client to connect to the program.
func (m *Command) Client() *http.InternalClient {
return m.Server.InternalClient().(*http.InternalClient)

26
topology/snapshot.go
View file

@ -68,9 +68,9 @@ func NewClusterSnapshot(noder Noder, hasher Hasher, replicas int) *ClusterSnapsh
//////////////////////////////////////////////////////////////////////////////
// shardToShardPartition returns the shard-partition that the given shard
// ShardToShardPartition returns the shard-partition that the given shard
// belongs to. NOTE: This is DIFFERENT from the key-partition.
func (c *ClusterSnapshot) shardToShardPartition(index string, shard uint64) int {
func (c *ClusterSnapshot) ShardToShardPartition(index string, shard uint64) int {
return dedupShardToShardPartition(index, shard, c.PartitionN)
}
@ -88,9 +88,14 @@ func dedupShardToShardPartition(index string, shard uint64, partitionN int) int
return int(h.Sum64() % uint64(partitionN))
}
// keyToKeyPartition returns the key-partition that the given key belongs to.
// IDToShardPartition returns the shard-partition that an id belongs to.
func (c *ClusterSnapshot) IDToShardPartition(index string, id uint64) int {
return c.ShardToShardPartition(index, id/ShardWidth)
}
// KeyToKeyPartition returns the key-partition that the given key belongs to.
// NOTE: The key-partition is DIFFERENT from the shard-partition.
func (c *ClusterSnapshot) keyToKeyPartition(index, key string) int {
func (c *ClusterSnapshot) KeyToKeyPartition(index, key string) int {
// Hash the bytes and mod by partition count.
h := fnv.New64a()
_, _ = h.Write([]byte(index))
@ -100,12 +105,17 @@ func (c *ClusterSnapshot) keyToKeyPartition(index, key string) int {
// ShardNodes returns a list of nodes that own a shard.
func (c *ClusterSnapshot) ShardNodes(index string, shard uint64) []*Node {
return c.PartitionNodes(c.shardToShardPartition(index, shard))
return c.PartitionNodes(c.ShardToShardPartition(index, shard))
}
// OwnsShard returns true if a host owns a fragment.
func (c *ClusterSnapshot) OwnsShard(nodeID string, index string, shard uint64) bool {
return Nodes(c.ShardNodes(index, shard)).ContainsID(nodeID)
}
// KeyNodes returns a list of nodes that own a key.
func (c *ClusterSnapshot) KeyNodes(index, key string) []*Node {
return c.PartitionNodes(c.keyToKeyPartition(index, key))
return c.PartitionNodes(c.KeyToKeyPartition(index, key))
}
// PartitionNodes returns a list of nodes that own the given partition.
@ -216,7 +226,7 @@ func (c *ClusterSnapshot) ReplicasForPrimary(primary int) (replicaNodeIDs, nonRe
func (c *ClusterSnapshot) ContainsShards(index string, availableShards *roaring.Bitmap, node *Node) []uint64 {
var shards []uint64
_ = availableShards.ForEach(func(i uint64) error {
p := c.shardToShardPartition(index, i)
p := c.ShardToShardPartition(index, i)
// Determine the nodes for partition.
nodes := c.PartitionNodes(p)
for _, n := range nodes {
@ -235,7 +245,7 @@ func (c *ClusterSnapshot) ContainsShards(index string, availableShards *roaring.
// replication. So with 4 nodes and 3-way replication, each node has 3/4 of
// the translation stores on it.
func (c *ClusterSnapshot) PrimaryForColKeyTranslation(index, key string) (primary int) {
partitionID := c.keyToKeyPartition(index, key)
partitionID := c.KeyToKeyPartition(index, key)
return c.PrimaryNodeIndex(partitionID)
}

8
translator_test.go
View file

@ -734,7 +734,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
for i, keys := range parts {
i, keys := i, keys
g.Go(func() error {
_, err := c.Nodes[i].API.CreateIndexKeys(ctx, "i", keys...)
_, err := c.GetNode(i).API.CreateIndexKeys(ctx, "i", keys...)
return err
})
}
@ -753,7 +753,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
}
// Obtain authoritative translations for the keys.
translations, err := c.Nodes[0].API.FindIndexKeys(ctx, "i", keyList...)
translations, err := c.GetCoordinator().API.FindIndexKeys(ctx, "i", keyList...)
if err != nil {
t.Errorf("obtaining authoritative translations: %v", err)
return
@ -820,7 +820,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
for i, keys := range parts {
i, keys := i, keys
g.Go(func() error {
_, err := c.Nodes[i].API.CreateFieldKeys(ctx, "i", "f", keys...)
_, err := c.GetNode(i).API.CreateFieldKeys(ctx, "i", "f", keys...)
return err
})
}
@ -839,7 +839,7 @@ func TestTranslation_Cluster_CreateFind(t *testing.T) {
}
// Obtain authoritative translations for the keys.
translations, err := c.Nodes[0].API.FindFieldKeys(ctx, "i", "f", keyList...)
translations, err := c.GetCoordinator().API.FindFieldKeys(ctx, "i", "f", keyList...)
if err != nil {
t.Errorf("obtaining authoritative translations: %v", err)
return

6
utils_internal_test.go
View file

@ -255,13 +255,13 @@ func (t *ClusterCluster) WriteTopology(path string, top *Topology) error {
}
func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) {
id := fmt.Sprintf("node%d", i)
uri := NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0))
node := &topology.Node{
ID: id,
URI: uri,
ID: id,
URI: uri,
IsCoordinator: i == 0,
}
// add URI to common