scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

first cut at removing all the shard/field/partition versioning

Browse source 
some cleanup needed

(cherry picked from commit bc9057f492)
This commit is contained in:
Matthew Jaffee 2023-01-02 12:19:09 -06:00 • committed by Joe Friedrich
parent 69a174412d
commit e09a9dca47
25 changed files with 436 additions and 2798 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
api.go
View file

@ -716,29 +716,6 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
}
}
func (api *API) getOrCreateShardVersion(ctx context.Context, indexName string, shard uint64) (int, error) {
tableName := dax.TableName(indexName)
shardNum := dax.ShardNum(shard)
// Here we assume that indexName is the string encoding of QualifiedTableID.
qtid, err := dax.QualifiedTableIDFromKey(indexName)
if err != nil {
return -1, errors.Wrap(err, "decoding qtid from key (indexName)")
}
version, found, err := api.holder.versionStore.ShardVersion(ctx, qtid, shardNum)
if err != nil {
return -1, errors.Wrap(err, "getting shard version")
} else if !found {
version = 0
api.server.logger.Printf("could not find version for shard: %s, %d, so creating 0", tableName, shardNum)
if err := api.holder.versionStore.AddShards(ctx, qtid, dax.NewVersionedShard(shardNum, version)); err != nil {
return -1, errors.Wrap(err, "adding shard 0")
}
}
return version, nil
}
// DeleteField removes the named field from the named index. If the index is not
// found, an error is returned. If the field is not found, it is ignored and no
// action is taken.
@ -3324,9 +3301,9 @@ var methodsNormal = map[apiMethod]struct{}{
apiDeleteDataframe: {},
}
func shardInShards(i dax.ShardNum, s dax.VersionedShards) bool {
func shardInShards(i dax.ShardNum, s dax.ShardNums) bool {
for _, o := range s {
if i == o.Num {
if i == o {
return true
}
}

56
api_directive.go
View file

@ -345,7 +345,7 @@ func (api *API) pushJobsTableKeys(ctx context.Context, jobs chan<- directiveJobT
jobs <- directiveJobTableKeys{
idx: idx,
tkey: tkey,
partition: partition.Num,
partition: partition,
}
}
}
@ -419,7 +419,7 @@ func (api *API) pushJobsFieldKeys(ctx context.Context, jobs chan<- directiveJobT
for _, field := range fields {
jobs <- directiveJobFieldKeys{
tkey: tkey,
field: field.Name,
field: field,
}
}
}
@ -503,7 +503,7 @@ func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType
for _, shard := range shards {
jobs <- directiveJobShards{
tkey: tkey,
shard: shard.Num,
shard: shard,
}
}
}
@ -710,11 +710,11 @@ func thingsAdded[K comparable](from []K, to []K) []K {
// partitionsComparer is used to compare the differences between two maps of
// table:[]partition.
type partitionsComparer struct {
from map[dax.TableKey]dax.VersionedPartitions
to map[dax.TableKey]dax.VersionedPartitions
from map[dax.TableKey]dax.PartitionNums
to map[dax.TableKey]dax.PartitionNums
}
func newPartitionsComparer(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.TableKey]dax.VersionedPartitions) *partitionsComparer {
func newPartitionsComparer(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) *partitionsComparer {
return &partitionsComparer{
from: from,
to: to,
@ -723,23 +723,23 @@ func newPartitionsComparer(from map[dax.TableKey]dax.VersionedPartitions, to map
// added returns the partitions which are present in `to` but not in `from`. The
// results remain in the format of a map of table:[]partition.
func (p *partitionsComparer) added() map[dax.TableKey]dax.VersionedPartitions {
func (p *partitionsComparer) added() map[dax.TableKey]dax.PartitionNums {
return partitionsAdded(p.from, p.to)
}
// removed returns the partitions which are present in `from` but not in `to`.
// The results remain in the format of a map of table:[]partition.
func (p *partitionsComparer) removed() map[dax.TableKey]dax.VersionedPartitions {
func (p *partitionsComparer) removed() map[dax.TableKey]dax.PartitionNums {
return partitionsAdded(p.to, p.from)
}
// partitionsAdded returns the partitions which are present in `to` but not in `from`.
func partitionsAdded(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.TableKey]dax.VersionedPartitions) map[dax.TableKey]dax.VersionedPartitions {
func partitionsAdded(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) map[dax.TableKey]dax.PartitionNums {
if from == nil {
return to
}
added := make(map[dax.TableKey]dax.VersionedPartitions)
added := make(map[dax.TableKey]dax.PartitionNums)
for tt, tps := range to {
fps, found := from[tt]
if !found {
@ -747,7 +747,7 @@ func partitionsAdded(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.T
continue
}
addedPartitions := dax.VersionedPartitions{}
addedPartitions := dax.PartitionNums{}
for i := range tps {
var found bool
for j := range fps {
@ -771,11 +771,11 @@ func partitionsAdded(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.T
// fieldsComparer is used to compare the differences between two maps of
// table:[]fieldVersion.
type fieldsComparer struct {
from map[dax.TableKey]dax.VersionedFields
to map[dax.TableKey]dax.VersionedFields
from map[dax.TableKey][]dax.FieldName
to map[dax.TableKey][]dax.FieldName
}
func newFieldsComparer(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]dax.VersionedFields) *fieldsComparer {
func newFieldsComparer(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) *fieldsComparer {
return &fieldsComparer{
from: from,
to: to,
@ -784,23 +784,23 @@ func newFieldsComparer(from map[dax.TableKey]dax.VersionedFields, to map[dax.Tab
// added returns the fields which are present in `to` but not in `from`. The
// results remain in the format of a map of table:[]field.
func (f *fieldsComparer) added() map[dax.TableKey]dax.VersionedFields {
func (f *fieldsComparer) added() map[dax.TableKey][]dax.FieldName {
return fieldsAdded(f.from, f.to)
}
// removed returns the fields which are present in `from` but not in `to`.
// The results remain in the format of a map of table:[]field.
func (f *fieldsComparer) removed() map[dax.TableKey]dax.VersionedFields {
func (f *fieldsComparer) removed() map[dax.TableKey][]dax.FieldName {
return fieldsAdded(f.to, f.from)
}
// fieldsAdded returns the fields which are present in `to` but not in `from`.
func fieldsAdded(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]dax.VersionedFields) map[dax.TableKey]dax.VersionedFields {
func fieldsAdded(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) map[dax.TableKey][]dax.FieldName {
if from == nil {
return to
}
added := make(map[dax.TableKey]dax.VersionedFields)
added := make(map[dax.TableKey][]dax.FieldName)
for tt, tps := range to {
fps, found := from[tt]
if !found {
@ -808,7 +808,7 @@ func fieldsAdded(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]
continue
}
addedFieldVersions := dax.VersionedFields{}
addedFieldVersions := []dax.FieldName{}
for i := range tps {
var found bool
for j := range fps {
@ -832,11 +832,11 @@ func fieldsAdded(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]
// shardsComparer is used to compare the differences between two maps of
// table:[]shardV.
type shardsComparer struct {
from map[dax.TableKey]dax.VersionedShards
to map[dax.TableKey]dax.VersionedShards
from map[dax.TableKey]dax.ShardNums
to map[dax.TableKey]dax.ShardNums
}
func newShardsComparer(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]dax.VersionedShards) *shardsComparer {
func newShardsComparer(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) *shardsComparer {
return &shardsComparer{
from: from,
to: to,
@ -845,23 +845,23 @@ func newShardsComparer(from map[dax.TableKey]dax.VersionedShards, to map[dax.Tab
// added returns the shards which are present in `to` but not in `from`. The
// results remain in the format of a map of table:[]shard.
func (s *shardsComparer) added() map[dax.TableKey]dax.VersionedShards {
func (s *shardsComparer) added() map[dax.TableKey]dax.ShardNums {
return shardsAdded(s.from, s.to)
}
// removed returns the shards which are present in `from` but not in `to`. The
// results remain in the format of a map of table:[]shard.
func (s *shardsComparer) removed() map[dax.TableKey]dax.VersionedShards {
func (s *shardsComparer) removed() map[dax.TableKey]dax.ShardNums {
return shardsAdded(s.to, s.from)
}
// shardsAdded returns the shards which are present in `to` but not in `from`.
func shardsAdded(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]dax.VersionedShards) map[dax.TableKey]dax.VersionedShards {
func shardsAdded(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) map[dax.TableKey]dax.ShardNums {
if from == nil {
return to
}
added := make(map[dax.TableKey]dax.VersionedShards)
added := make(map[dax.TableKey]dax.ShardNums)
for tt, tss := range to {
fss, found := from[tt]
if !found {
@ -869,7 +869,7 @@ func shardsAdded(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]
continue
}
addedShards := dax.VersionedShards{}
addedShards := dax.ShardNums{}
for i := range tss {
var found bool
for j := range fss {
@ -892,7 +892,7 @@ func shardsAdded(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]
// createTableAndFields creates the FeatureBase Tables and Fields provided in
// the dax.Directive format.
func (api *API) createTableAndFields(tbl *dax.QualifiedTable, partitions dax.VersionedPartitions) error {
func (api *API) createTableAndFields(tbl *dax.QualifiedTable, partitions dax.PartitionNums) error {
cim := &CreateIndexMessage{
Index: string(tbl.Key()),
CreatedAt: 0,

5
cluster.go
View file

@ -74,7 +74,6 @@ type cluster struct { // nolint: maligned
partitionAssigner string
serverlessStorage *storage.ResourceManager
versionStore dax.VersionStore
// isComputeNode is set to true if this node is running as a DAX compute
// node.
@ -1013,9 +1012,9 @@ type TransactionMessage struct {
Action string
}
func intInPartitions(i int, s dax.VersionedPartitions) bool {
func intInPartitions(i int, s dax.PartitionNums) bool {
for _, a := range s {
if int(a.Num) == i {
if int(a) == i {
return true
}
}

768
dax/boltdb/versionstore.go
View file

@ -1,768 +0,0 @@
package boltdb
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"strconv"
"strings"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/inmem"
"github.com/featurebasedb/featurebase/v3/errors"
"github.com/featurebasedb/featurebase/v3/logger"
)
var (
bucketTables = Bucket("versionStoreTables")
bucketShards = Bucket("versionStoreShards")
bucketTableKeys = Bucket("versionStoreTableKeys")
bucketFieldKeys = Bucket("versionStoreFieldKeys")
)
// VersionStoreBuckets defines the buckets used by this package. It can be
// called during setup to create the buckets ahead of time.
var VersionStoreBuckets []Bucket = []Bucket{
bucketTables,
bucketShards,
bucketTableKeys,
bucketFieldKeys,
}
// Ensure type implements interface.
var _ dax.VersionStore = (*VersionStore)(nil)
// VersionStore manages all version info for shard, table keys, and field keys.
type VersionStore struct {
db *DB
logger logger.Logger
}
// NewVersionStore returns a new instance of VersionStore with default values.
func NewVersionStore(db *DB, logger logger.Logger) *VersionStore {
return &VersionStore{
db: db,
logger: logger,
}
}
func (s *VersionStore) AddTable(ctx context.Context, qtid dax.QualifiedTableID) error {
tx, err := s.db.BeginTx(ctx, true)
if err != nil {
return errors.Wrap(err, "getting transaction")
}
defer tx.Rollback()
bkt := tx.Bucket(bucketTables)
if bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketTables)
}
if val := bkt.Get(tableKey(qtid)); val != nil {
return dax.NewErrTableIDExists(qtid)
}
// The assumption is that we may store information about the table (other
// than just the fact that it exists). So for now, the value is an empty
// JSON object.
val := []byte("{}")
if err := bkt.Put(tableKey(qtid), val); err != nil {
return errors.Wrap(err, "putting table")
}
// Add the table to the "table index" of the other buckets.
//
// Shards
if bkt := tx.Bucket(bucketShards); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketShards)
} else if err := bkt.Put(tableKey(qtid), val); err != nil {
return errors.Wrap(err, "putting table into shards")
}
// TableKeys.
if bkt := tx.Bucket(bucketTableKeys); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketTableKeys)
} else if err := bkt.Put(tableKey(qtid), val); err != nil {
return errors.Wrap(err, "putting table into table keys")
}
// FieldKeys.
if bkt := tx.Bucket(bucketFieldKeys); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketFieldKeys)
} else if err := bkt.Put(tableKey(qtid), val); err != nil {
return errors.Wrap(err, "putting table into field keys")
}
return tx.Commit()
}
func (s *VersionStore) RemoveTable(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedShards, dax.VersionedPartitions, error) {
tx, err := s.db.BeginTx(ctx, true)
if err != nil {
return nil, nil, err
}
defer tx.Rollback()
// Get the shards and partitions before deleting by table.
shards, err := s.getShards(ctx, tx, qtid)
if err != nil {
return nil, nil, err
}
partitions, err := s.getPartitions(ctx, tx, qtid)
if err != nil {
return nil, nil, err
}
if err := removeTable(ctx, tx, qtid); err != nil {
return nil, nil, err
}
if err := tx.Commit(); err != nil {
return nil, nil, err
}
return shards, partitions, nil
}
func removeTable(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID) error {
// Tables.
if bkt := tx.Bucket(bucketTables); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketTables)
} else if err := bkt.Delete(tableKey(qtid)); err != nil {
return errors.Wrap(err, "deleting table")
}
// Shards.
if bkt := tx.Bucket(bucketShards); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketShards)
} else if err := bkt.Delete(tableKey(qtid)); err != nil {
return errors.Wrap(err, "deleting table in shards")
} else if err := deleteByPrefix(tx, bucketShards, []byte(fmt.Sprintf(prefixFmtShards, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))); err != nil {
return errors.Wrap(err, "deleting shards for table")
}
// TableKeys.
if bkt := tx.Bucket(bucketTableKeys); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketTableKeys)
} else if err := bkt.Delete(tableKey(qtid)); err != nil {
return errors.Wrap(err, "deleting table in table keys")
} else if err := deleteByPrefix(tx, bucketTableKeys, []byte(fmt.Sprintf(prefixFmtTableKeys, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))); err != nil {
return errors.Wrap(err, "deleting table keys for table")
}
// FieldKeys.
if bkt := tx.Bucket(bucketFieldKeys); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketFieldKeys)
} else if err := bkt.Delete(tableKey(qtid)); err != nil {
return errors.Wrap(err, "deleting table in field keys")
} else if err := deleteByPrefix(tx, bucketFieldKeys, []byte(fmt.Sprintf(prefixFmtFieldKeys, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))); err != nil {
return errors.Wrap(err, "deleting field keys for table")
}
return nil
}
func deleteByPrefix(tx *Tx, bucket Bucket, prefix []byte) error {
bkt := tx.Bucket(bucket)
cursor := bkt.Cursor()
// Deleting keys within the for loop seems to cause Next() to skip the next
// matching key because the Delete() call pops the item and effectively
// moves the cursor forward. Then calling Next() skips the item that was
// being pointed to after the delete. So, we're going to make a list of keys
// to delete, and then delete them outside of the cursor logic.
var keysToDelete [][]byte
for k, _ := cursor.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = cursor.Next() {
keysToDelete = append(keysToDelete, k)
}
for _, k := range keysToDelete {
if err := bkt.Delete(k); err != nil {
return errors.Wrapf(err, "deleting key: %s", k)
}
}
return nil
}
func (s *VersionStore) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.VersionedShard) error {
tx, err := s.db.BeginTx(ctx, true)
if err != nil {
return errors.Wrap(err, "getting transaction")
}
defer tx.Rollback()
for _, shard := range shards {
if err := createShard(ctx, tx, qtid, shard); err != nil {
return errors.Wrap(err, "creating shard")
}
}
return tx.Commit()
}
func createShard(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, shard dax.VersionedShard) error {
// TODO: validate data more formally
if shard.Version < 0 {
return errors.New(errors.ErrUncoded, fmt.Sprintf("invalid shard version: %d", shard.Version))
}
bkt := tx.Bucket(bucketShards)
if bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketShards)
}
// Ensure the table exists.
if val := bkt.Get(tableKey(qtid)); val == nil {
return dax.NewErrTableIDDoesNotExist(qtid)
}
vsn := make([]byte, 8)
binary.LittleEndian.PutUint64(vsn, uint64(shard.Version))
return bkt.Put(shardKey(qtid, shard.Num), vsn)
}
func (s *VersionStore) Shards(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedShards, bool, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, false, errors.Wrap(err, "getting tx")
}
defer tx.Rollback()
shards, err := s.getShards(ctx, tx, qtid)
if err != nil {
return nil, false, errors.Wrap(err, "getting shards")
}
return shards, true, nil
}
func (s *VersionStore) getShards(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID) (dax.VersionedShards, error) {
c := tx.Bucket(bucketShards).Cursor()
// Deserialize rows into Shard objects.
shards := make(dax.VersionedShards, 0)
prefix := []byte(fmt.Sprintf(prefixFmtShards, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
if v == nil {
s.logger.Printf("nil value for key: %s", k)
continue
}
var shard dax.VersionedShard
shardNum, err := keyShardNum(k)
if err != nil {
return nil, errors.Wrapf(err, "getting shardNum from key: %v", k)
}
shard.Num = shardNum
shard.Version = int(binary.LittleEndian.Uint64(v))
shards = append(shards, shard)
}
return shards, nil
}
// ShardVersion return the current version for the given table/shardNum.
// If a version is not being tracked, it returns a bool value of false.
func (s *VersionStore) ShardVersion(ctx context.Context, qtid dax.QualifiedTableID, shardNum dax.ShardNum) (int, bool, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return -1, false, err
}
defer tx.Rollback()
return getShardVersion(ctx, tx, qtid, shardNum)
}
func getShardVersion(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, shardNum dax.ShardNum) (int, bool, error) {
version := -1
bkt := tx.Bucket(bucketShards)
if bkt == nil {
return version, false, errors.Errorf(ErrFmtBucketNotFound, bucketShards)
}
b := bkt.Get(shardKey(qtid, shardNum))
if b == nil {
return version, false, nil
}
version = int(binary.LittleEndian.Uint64(b))
return version, true, nil
}
func (s *VersionStore) ShardTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, errors.Wrap(err, "beginning tx")
}
defer tx.Rollback()
return s.getTableIDs(ctx, tx, qual, bucketShards)
}
func (s *VersionStore) getTableIDs(ctx context.Context, tx *Tx, qual dax.TableQualifier, bucket Bucket) (dax.TableIDs, error) {
c := tx.Bucket(bucket).Cursor()
// Deserialize rows into Tables objects.
tableIDs := make(dax.TableIDs, 0)
prefix := []byte(fmt.Sprintf(prefixFmtTables, qual.OrganizationID, qual.DatabaseID))
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
if v == nil {
s.logger.Printf("nil value for key: %s", k)
continue
}
var tableID dax.TableID
tableID, err := keyTableID(k)
if err != nil {
return nil, errors.Wrapf(err, "getting table name from key: %v", k)
}
tableIDs = append(tableIDs, tableID)
}
return tableIDs, nil
}
func (s *VersionStore) bucketTables(ctx context.Context, bucket Bucket) ([]dax.QualifiedTableID, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, errors.Wrap(err, "beginning tx")
}
defer tx.Rollback()
c := tx.Bucket(bucket).Cursor()
// Deserialize rows into Tables objects.
qtids := make([]dax.QualifiedTableID, 0)
prefix := []byte(prefixTables)
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
if v == nil {
s.logger.Printf("nil value for key: %s", k)
continue
}
qtid, err := keyQualifiedTableID(k)
if err != nil {
return nil, errors.Wrapf(err, "getting qualified table id from key: %v", k)
}
qtids = append(qtids, qtid)
}
return qtids, nil
}
// AddPartitions adds new partitions to be managed by VersionStore. It returns
// the number of partitions added or an error.
func (s *VersionStore) AddPartitions(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.VersionedPartition) error {
tx, err := s.db.BeginTx(ctx, true)
if err != nil {
return errors.Wrap(err, "getting transaction")
}
defer tx.Rollback()
for _, partition := range partitions {
if err := createPartition(ctx, tx, qtid, partition); err != nil {
return errors.Wrap(err, "creating partition")
}
}
return tx.Commit()
}
func createPartition(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, partition dax.VersionedPartition) error {
// TODO: validate data more formally
if partition.Version < 0 {
return errors.New(errors.ErrUncoded, fmt.Sprintf("invalid partition version: %d", partition.Version))
}
bkt := tx.Bucket(bucketTableKeys)
if bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketTableKeys)
}
// Ensure the table exists.
if val := bkt.Get(tableKey(qtid)); val == nil {
return dax.NewErrTableIDDoesNotExist(qtid)
}
vsn := make([]byte, 8)
binary.LittleEndian.PutUint64(vsn, uint64(partition.Version))
return bkt.Put(partitionKey(qtid, partition.Num), vsn)
}
func (s *VersionStore) Partitions(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedPartitions, bool, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, false, errors.Wrap(err, "getting tx")
}
defer tx.Rollback()
partitions, err := s.getPartitions(ctx, tx, qtid)
if err != nil {
return nil, false, errors.Wrap(err, "getting partitions")
}
return partitions, true, nil
}
func (s *VersionStore) getPartitions(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID) (dax.VersionedPartitions, error) {
c := tx.Bucket(bucketTableKeys).Cursor()
// Deserialize rows into Partition objects.
partitions := make(dax.VersionedPartitions, 0)
prefix := []byte(fmt.Sprintf(prefixFmtTableKeys, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
if v == nil {
s.logger.Printf("nil value for key: %s", k)
continue
}
var partition dax.VersionedPartition
partitionNum, err := keyPartitionNum(k)
if err != nil {
return nil, errors.Wrapf(err, "getting partitionNum from key: %v", k)
}
partition.Num = partitionNum
partition.Version = int(binary.LittleEndian.Uint64(v))
partitions = append(partitions, partition)
}
return partitions, nil
}
func (s *VersionStore) PartitionVersion(ctx context.Context, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) (int, bool, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return -1, false, err
}
defer tx.Rollback()
return getPartitionVersion(ctx, tx, qtid, partitionNum)
}
func getPartitionVersion(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) (int, bool, error) {
version := -1
bkt := tx.Bucket(bucketTableKeys)
if bkt == nil {
return version, false, errors.Errorf(ErrFmtBucketNotFound, bucketTableKeys)
}
b := bkt.Get(partitionKey(qtid, partitionNum))
if b == nil {
return version, false, nil
}
version = int(binary.LittleEndian.Uint64(b))
return version, true, nil
}
func (s *VersionStore) PartitionTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, errors.Wrap(err, "beginning tx")
}
defer tx.Rollback()
return s.getTableIDs(ctx, tx, qual, bucketTableKeys)
}
// AddFields adds new fields to be managed by VersionStore. It returns the
// number of fields added or an error.
func (s *VersionStore) AddFields(ctx context.Context, qtid dax.QualifiedTableID, fields ...dax.VersionedField) error {
tx, err := s.db.BeginTx(ctx, true)
if err != nil {
return err
}
defer tx.Rollback()
for _, field := range fields {
if err := createFieldVersion(ctx, tx, qtid, field); err != nil {
return errors.Wrap(err, "creating field version")
}
}
return tx.Commit()
}
func createFieldVersion(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, field dax.VersionedField) error {
// TODO: validate data more formally
if field.Version < 0 {
return errors.New(errors.ErrUncoded, fmt.Sprintf("invalid field version: %d", field.Version))
}
bkt := tx.Bucket(bucketFieldKeys)
if bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketFieldKeys)
}
// Ensure the table exists.
if val := bkt.Get(tableKey(qtid)); val == nil {
return dax.NewErrTableIDDoesNotExist(qtid)
}
vsn := make([]byte, 8)
binary.LittleEndian.PutUint64(vsn, uint64(field.Version))
return bkt.Put(fieldKey(qtid, field.Name), vsn)
}
func (s *VersionStore) Fields(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedFields, bool, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, false, errors.Wrap(err, "getting tx")
}
defer tx.Rollback()
fields, err := s.getFields(ctx, tx, qtid)
if err != nil {
return nil, false, errors.Wrap(err, "getting fields")
}
return fields, true, nil
}
func (s *VersionStore) getFields(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID) (dax.VersionedFields, error) {
c := tx.Bucket(bucketFieldKeys).Cursor()
// Deserialize rows into FieldVersion objects.
fieldVersions := make(dax.VersionedFields, 0)
prefix := []byte(fmt.Sprintf(prefixFmtFieldKeys, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
if v == nil {
s.logger.Printf("nil value for key: %s", k)
continue
}
var fieldVersion dax.VersionedField
fieldName, err := keyFieldName(k)
if err != nil {
return nil, errors.Wrapf(err, "getting partitionNum from key: %v", k)
}
fieldVersion.Name = fieldName
fieldVersion.Version = int(binary.LittleEndian.Uint64(v))
fieldVersions = append(fieldVersions, fieldVersion)
}
return fieldVersions, nil
}
func (s *VersionStore) FieldVersion(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName) (int, bool, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return -1, false, err
}
defer tx.Rollback()
return getFieldVersion(ctx, tx, qtid, field)
}
func getFieldVersion(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, field dax.FieldName) (int, bool, error) {
version := -1
bkt := tx.Bucket(bucketFieldKeys)
if bkt == nil {
return version, false, errors.Errorf(ErrFmtBucketNotFound, bucketFieldKeys)
}
b := bkt.Get(fieldKey(qtid, field))
if b == nil {
return version, false, nil
}
version = int(binary.LittleEndian.Uint64(b))
return version, true, nil
}
func (s *VersionStore) FieldTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, errors.Wrap(err, "beginning tx")
}
defer tx.Rollback()
return s.getTableIDs(ctx, tx, qual, bucketFieldKeys)
}
// Copy returns an in-memory copy of VersionStore.
func (s *VersionStore) Copy(ctx context.Context) (dax.VersionStore, error) {
new := inmem.NewVersionStore()
// shards.
qtids, err := s.bucketTables(ctx, bucketShards)
if err != nil {
return nil, errors.Wrap(err, "getting shard tables")
}
for _, qtid := range qtids {
shards, found, err := s.Shards(ctx, qtid)
if err != nil {
return nil, errors.Wrap(err, "getting shards")
} else if !found {
continue
}
_ = new.AddTable(ctx, qtid)
new.AddShards(ctx, qtid, shards...)
}
// tableKeys.
qtids, err = s.bucketTables(ctx, bucketTableKeys)
if err != nil {
return nil, errors.Wrap(err, "getting table key tables")
}
for _, qtid := range qtids {
partitions, found, err := s.Partitions(ctx, qtid)
if err != nil {
return nil, errors.Wrap(err, "getting partitions")
} else if !found {
continue
}
_ = new.AddTable(ctx, qtid)
new.AddPartitions(ctx, qtid, partitions...)
}
// fieldKeys.
qtids, err = s.bucketTables(ctx, bucketFieldKeys)
if err != nil {
return nil, errors.Wrap(err, "getting field key tables")
}
for _, qtid := range qtids {
fields, found, err := s.Fields(ctx, qtid)
if err != nil {
return nil, errors.Wrap(err, "getting fields")
} else if !found {
continue
}
_ = new.AddTable(ctx, qtid)
new.AddFields(ctx, qtid, fields...)
}
return new, nil
}
/////////////////////////////////////////////////////////
const (
prefixShards = "shards/"
prefixFmtShards = prefixShards + "%s/%s/%s/"
prefixTableKeys = "tablekeys/"
prefixFmtTableKeys = prefixTableKeys + "%s/%s/%s/"
prefixFieldKeys = "fieldkeys/"
prefixFmtFieldKeys = prefixFieldKeys + "%s/%s/%s/"
prefixTables = "tables/"
prefixFmtTables = prefixTables + "%s/%s/"
)
// tableKey returns a key based on table name.
func tableKey(qtid dax.QualifiedTableID) []byte {
qual := qtid.TableQualifier
key := fmt.Sprintf(prefixFmtTables+"%s", qual.OrganizationID, qual.DatabaseID, qtid.ID)
return []byte(key)
}
// keyTableID gets the table ID out of the key.
func keyTableID(key []byte) (dax.TableID, error) {
parts := strings.Split(string(key), "/")
if len(parts) != 4 {
return "", errors.New(errors.ErrUncoded, "table key format expected: `tables/orgID/dbID/tableID`")
}
return dax.TableID(parts[3]), nil
}
// keyQualifiedTableID gets the qualified table ID out of the key.
func keyQualifiedTableID(key []byte) (dax.QualifiedTableID, error) {
parts := strings.Split(string(key), "/")
if len(parts) != 4 {
return dax.QualifiedTableID{}, errors.New(errors.ErrUncoded, "table key format expected: `tables/orgID/dbID/tableID`")
}
return dax.NewQualifiedTableID(
dax.NewTableQualifier(dax.OrganizationID(parts[1]), dax.DatabaseID(parts[2])),
dax.TableID(parts[3]),
), nil
}
// shardKey returns a key based on table and shard.
func shardKey(qtid dax.QualifiedTableID, shard dax.ShardNum) []byte {
key := fmt.Sprintf(prefixFmtShards+"%d", qtid.OrganizationID, qtid.DatabaseID, qtid.ID, shard)
return []byte(key)
}
// keyShardNum gets the shardNum out of the key.
func keyShardNum(key []byte) (dax.ShardNum, error) {
parts := strings.Split(string(key), "/")
if len(parts) != 5 {
return 0, errors.New(errors.ErrUncoded, "shard key format expected: `shards/orgID/dbID/table/shard`")
}
intVar, err := strconv.Atoi(parts[4])
if err != nil {
return 0, errors.Wrapf(err, "converting string to shardNum: %s", parts[4])
}
return dax.ShardNum(intVar), nil
}
// partitionKey returns a key based on table and partition.
func partitionKey(qtid dax.QualifiedTableID, partition dax.PartitionNum) []byte {
key := fmt.Sprintf(prefixFmtTableKeys+"%d", qtid.OrganizationID, qtid.DatabaseID, qtid.ID, partition)
return []byte(key)
}
// keyPartitionNum gets the partitionNum out of the key.
func keyPartitionNum(key []byte) (dax.PartitionNum, error) {
parts := strings.Split(string(key), "/")
if len(parts) != 5 {
return 0, errors.New(errors.ErrUncoded, "partition key format expected: `tablekeys/orgID/dbID/table/partition`")
}
intVar, err := strconv.Atoi(parts[4])
if err != nil {
return 0, errors.Wrapf(err, "converting string to partitionNum: %s", parts[4])
}
return dax.PartitionNum(intVar), nil
}
// fieldKey returns a key based on table and field.
func fieldKey(qtid dax.QualifiedTableID, field dax.FieldName) []byte {
key := fmt.Sprintf(prefixFmtFieldKeys+"%s", qtid.OrganizationID, qtid.DatabaseID, qtid.ID, field)
return []byte(key)
}
// keyFieldName gets the fieldName out of the key.
func keyFieldName(key []byte) (dax.FieldName, error) {
parts := strings.Split(string(key), "/")
if len(parts) != 5 {
return "", errors.New(errors.ErrUncoded, "field key format expected: `fieldkeys/orgID/dbID/table/field`")
}
return dax.FieldName(parts[4]), nil
}

388
dax/boltdb/versionstore_test.go
View file

@ -1,388 +0,0 @@
package boltdb_test
import (
"context"
"fmt"
"sort"
"testing"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/boltdb"
testbolt "github.com/featurebasedb/featurebase/v3/dax/test/boltdb"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/stretchr/testify/assert"
)
func TestVersionStore(t *testing.T) {
db := testbolt.MustOpenDB(t)
defer testbolt.MustCloseDB(t, db)
ctx := context.Background()
t.Cleanup(func() {
testbolt.CleanupDB(t, db.Path())
})
orgID := dax.OrganizationID("acme")
dbID := dax.DatabaseID("db1")
qual := dax.NewTableQualifier(orgID, dbID)
// Initialize the buckets.
assert.NoError(t, db.InitializeBuckets(boltdb.VersionStoreBuckets...))
t.Run("Tables", func(t *testing.T) {
vs := boltdb.NewVersionStore(db, logger.NopLogger)
qtids := newQualifiedTableIDs(t, qual, 3)
qtid1 := qtids[0]
qtid2 := qtids[1]
qtid3 := qtids[2]
defer vs.RemoveTable(ctx, qtid1)
defer vs.RemoveTable(ctx, qtid2)
defer vs.RemoveTable(ctx, qtid3)
// Add table 1.
assert.NoError(t, vs.AddTable(ctx, qtid1))
// Add table 2.
assert.NoError(t, vs.AddTable(ctx, qtid2))
// Add table 3.
assert.NoError(t, vs.AddTable(ctx, qtid3))
})
t.Run("Shards", func(t *testing.T) {
vs := boltdb.NewVersionStore(db, logger.NopLogger)
qtids := newQualifiedTableIDs(t, qual, 3)
qtid1 := qtids[0]
qtid2 := qtids[1]
qtid3 := qtids[2]
// Add tables.
assert.NoError(t, vs.AddTable(ctx, qtid1))
assert.NoError(t, vs.AddTable(ctx, qtid2))
assert.NoError(t, vs.AddTable(ctx, qtid3))
defer vs.RemoveTable(ctx, qtid1)
defer vs.RemoveTable(ctx, qtid2)
defer vs.RemoveTable(ctx, qtid3)
// Create some shards to insert into the table.
shards := make(dax.VersionedShards, 3)
for i := range shards {
shards[i] = dax.VersionedShard{
Num: dax.ShardNum(i),
Version: i * 2,
}
}
// Add shards to table 1.
{
err := vs.AddShards(ctx, qtid1, shards...)
assert.NoError(t, err)
}
// Add shards to table 2.
{
err := vs.AddShards(ctx, qtid2, shards...)
assert.NoError(t, err)
}
// Fetch a shard and compare.
{
version, found, err := vs.ShardVersion(ctx, qtid1, 2)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, 4, version)
}
// Fetch all shards and compare.
{
shrds, found, err := vs.Shards(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, shards, shrds)
}
// Fetch tables.
{
tblIDs, err := vs.ShardTables(ctx, qual)
assert.NoError(t, err)
exp := dax.TableIDs{qtid1.ID, qtid2.ID, qtid3.ID}
assert.Equal(t, exp, tblIDs)
}
// Remove table 1.
{
shards, partitions, err := vs.RemoveTable(ctx, qtid1)
assert.NoError(t, err)
assert.Equal(t, shards, shards)
assert.Equal(t, dax.VersionedPartitions{}, partitions)
}
// Fetch all shards and compare.
{
shrds, found, err := vs.Shards(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, dax.VersionedShards{}, shrds)
}
})
t.Run("Partitions", func(t *testing.T) {
vs := boltdb.NewVersionStore(db, logger.NopLogger)
// Create some partitions to insert into the table.
partitions := make(dax.VersionedPartitions, 3)
for i := range partitions {
partitions[i] = dax.VersionedPartition{
Num: dax.PartitionNum(i),
Version: i * 2,
}
}
qtids := newQualifiedTableIDs(t, qual, 2)
qtid1 := qtids[0]
qtid2 := qtids[1]
// Add tables.
assert.NoError(t, vs.AddTable(ctx, qtid1))
assert.NoError(t, vs.AddTable(ctx, qtid2))
defer vs.RemoveTable(ctx, qtid1)
defer vs.RemoveTable(ctx, qtid2)
// Add partitions to table 1.
{
err := vs.AddPartitions(ctx, qtid1, partitions...)
assert.NoError(t, err)
}
// Add partitions to table 2.
{
err := vs.AddPartitions(ctx, qtid2, partitions...)
assert.NoError(t, err)
}
// Fetch a partition and compare.
{
version, found, err := vs.PartitionVersion(ctx, qtid1, 2)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, 4, version)
}
// Fetch all partitions and compare.
{
parts, found, err := vs.Partitions(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, partitions, parts)
}
// Fetch tables.
{
tblIDs, err := vs.PartitionTables(ctx, qual)
assert.NoError(t, err)
exp := dax.TableIDs{qtid1.ID, qtid2.ID}
assert.Equal(t, exp, tblIDs)
}
// Remove table 1.
{
shards, partitions, err := vs.RemoveTable(ctx, qtid1)
assert.NoError(t, err)
assert.Equal(t, dax.VersionedShards{}, shards)
assert.Equal(t, partitions, partitions)
}
// Fetch all partitions and compare.
{
parts, found, err := vs.Partitions(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, dax.VersionedPartitions{}, parts)
}
})
t.Run("FieldVersions", func(t *testing.T) {
vs := boltdb.NewVersionStore(db, logger.NopLogger)
qtids := newQualifiedTableIDs(t, qual, 2)
qtid1 := qtids[0]
qtid2 := qtids[1]
// Add tables.
assert.NoError(t, vs.AddTable(ctx, qtid1))
assert.NoError(t, vs.AddTable(ctx, qtid2))
defer vs.RemoveTable(ctx, qtid1)
defer vs.RemoveTable(ctx, qtid2)
// Create some fieldVersions to insert into the table.
fieldVersions := make(dax.VersionedFields, 3)
for i := range fieldVersions {
fieldVersions[i] = dax.VersionedField{
Name: dax.FieldName(fmt.Sprintf("fld-%d", i)),
Version: i * 2,
}
}
// Add fieldVersions to table 1.
{
err := vs.AddFields(ctx, qtid1, fieldVersions...)
assert.NoError(t, err)
}
// Add fieldVersions to table 2.
{
err := vs.AddFields(ctx, qtid2, fieldVersions...)
assert.NoError(t, err)
}
// Fetch a fieldVersion and compare.
{
version, found, err := vs.FieldVersion(ctx, qtid1, dax.FieldName("fld-2"))
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, 4, version)
}
// Fetch all fieldVersions and compare.
{
flds, found, err := vs.Fields(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, fieldVersions, flds)
}
// Fetch tables.
{
tblIDs, err := vs.FieldTables(ctx, qual)
assert.NoError(t, err)
exp := dax.TableIDs{qtid1.ID, qtid2.ID}
assert.Equal(t, exp, tblIDs)
}
// Remove table 1.
{
shards, partitions, err := vs.RemoveTable(ctx, qtid1)
assert.NoError(t, err)
assert.Equal(t, dax.VersionedShards{}, shards)
assert.Equal(t, dax.VersionedPartitions{}, partitions)
}
// Fetch all fieldVersions and compare.
{
flds, found, err := vs.Fields(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, dax.VersionedFields{}, flds)
}
})
t.Run("Copy", func(t *testing.T) {
vs := boltdb.NewVersionStore(db, logger.NopLogger)
qtids := newQualifiedTableIDs(t, qual, 1)
qtid1 := qtids[0]
// Add tables.
assert.NoError(t, vs.AddTable(ctx, qtid1))
defer vs.RemoveTable(ctx, qtid1)
// Create some shards to insert into the table.
shards := make(dax.VersionedShards, 3)
for i := range shards {
shards[i] = dax.VersionedShard{
Num: dax.ShardNum(i),
Version: i * 2,
}
}
// Create some partitions to insert into the table.
partitions := make(dax.VersionedPartitions, 3)
for i := range partitions {
partitions[i] = dax.VersionedPartition{
Num: dax.PartitionNum(i),
Version: i * 2,
}
}
// Create some fieldVersions to insert into the table.
fieldVersions := make(dax.VersionedFields, 3)
for i := range fieldVersions {
fieldVersions[i] = dax.VersionedField{
Name: dax.FieldName(fmt.Sprintf("fld-%d", i)),
Version: i * 2,
}
}
// Add shards to table 1.
{
err := vs.AddShards(ctx, qtid1, shards...)
assert.NoError(t, err)
}
// Add partitions to table 1.
{
err := vs.AddPartitions(ctx, qtid1, partitions...)
assert.NoError(t, err)
}
// Add fieldVersions to table 1.
{
err := vs.AddFields(ctx, qtid1, fieldVersions...)
assert.NoError(t, err)
}
copy, err := vs.Copy(ctx)
assert.NoError(t, err)
// Fetch a shard and compare.
{
version, found, err := copy.ShardVersion(ctx, qtid1, 2)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, 4, version)
}
// Fetch all partitions and compare.
{
parts, found, err := copy.Partitions(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, partitions, parts)
}
// Fetch all fieldVersions and compare.
{
flds, found, err := copy.Fields(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, fieldVersions, flds)
}
})
}
// newQualifiedTableIDs is a test helper function which generates a slice of n
// qtid. The entries in the slice will be ordered by TableID.
func newQualifiedTableIDs(t *testing.T, qual dax.TableQualifier, n int) []dax.QualifiedTableID {
t.Helper()
qtids := make([]dax.QualifiedTableID, n)
for i := range qtids {
tbl := dax.NewTable("testvstore")
tbl.CreateID()
qtids[i] = dax.NewQualifiedTableID(
qual,
tbl.ID,
)
}
// sort the qtids by ID
sort.Slice(qtids, func(i, j int) bool {
return qtids[i].ID < qtids[j].ID
})
return qtids
}

30
dax/directive.go
View file

@ -1,5 +1,7 @@
package dax
import "context"
// Directive contains the instructions, sent from MDS, which a compute node is
// to follow. A Directive is typically JSON-encoded and POSTed to a compute
// node's `/directive` endpoint.
@ -19,6 +21,10 @@ type Directive struct {
Version uint64 `json:"version"`
}
type DirectiveVersion interface {
Increment(ctx context.Context, delta uint64) (uint64, error)
}
// DirectiveMethod is used to tell the compute node how it should handle the
// Directive.
type DirectiveMethod string
@ -57,9 +63,9 @@ func (d *Directive) Table(qtid QualifiedTableID) (*QualifiedTable, error) {
// compute node is responsible. It assumes that the Directive does not contain
// more than one ComputeRole for the same table; in that case, we would need to
// return the union of Shards.
func (d *Directive) ComputeShards(tbl TableKey) VersionedShards {
func (d *Directive) ComputeShards(tbl TableKey) ShardNums {
if d == nil || d.ComputeRoles == nil {
return VersionedShards{}
return nil
}
for _, cr := range d.ComputeRoles {
@ -68,14 +74,14 @@ func (d *Directive) ComputeShards(tbl TableKey) VersionedShards {
}
}
return VersionedShards{}
return nil
}
// ComputeShardsMap returns a map of table to shards. It assumes that the
// Directive does not contain more than one ComputeRole for the same table; in
// that case, we would need to return the union of Shards.
func (d *Directive) ComputeShardsMap() map[TableKey]VersionedShards {
m := make(map[TableKey]VersionedShards)
func (d *Directive) ComputeShardsMap() map[TableKey]ShardNums {
m := make(map[TableKey]ShardNums)
if d == nil || d.ComputeRoles == nil {
return m
}
@ -91,9 +97,9 @@ func (d *Directive) ComputeShardsMap() map[TableKey]VersionedShards {
// which this translate node is responsible. It assumes that the Directive does
// not contain more than one TranslateRole for the same table; in that case, we
// would need to return the union of Shards.
func (d *Directive) TranslatePartitions(tbl TableKey) VersionedPartitions {
func (d *Directive) TranslatePartitions(tbl TableKey) PartitionNums {
if d == nil || d.TranslateRoles == nil {
return VersionedPartitions{}
return PartitionNums{}
}
for _, tr := range d.TranslateRoles {
@ -101,14 +107,14 @@ func (d *Directive) TranslatePartitions(tbl TableKey) VersionedPartitions {
return tr.Partitions
}
}
return VersionedPartitions{}
return PartitionNums{}
}
// TranslatePartitionsMap returns a map of table to partitions. It assumes that
// the Directive does not contain more than one TranslateRole for the same
// table; in that case, we would need to return the union of Partitions.
func (d *Directive) TranslatePartitionsMap() map[TableKey]VersionedPartitions {
m := make(map[TableKey]VersionedPartitions)
func (d *Directive) TranslatePartitionsMap() map[TableKey]PartitionNums {
m := make(map[TableKey]PartitionNums)
if d == nil || d.TranslateRoles == nil {
return m
}
@ -129,8 +135,8 @@ func (d *Directive) TranslatePartitionsMap() map[TableKey]VersionedPartitions {
// TranslateFieldsMap returns a map of table to fields. It assumes that
// the Directive does not contain more than one TranslateRole for the same
// table; in that case, we would need to return the union of FieldValues.
func (d *Directive) TranslateFieldsMap() map[TableKey]VersionedFields {
m := make(map[TableKey]VersionedFields)
func (d *Directive) TranslateFieldsMap() map[TableKey][]FieldName {
m := make(map[TableKey][]FieldName)
if d == nil || d.TranslateRoles == nil {
return m
}

2
dax/inmem/inmem.go
View file

@ -1,2 +0,0 @@
// Package inmem contains the in-memory implementation of the dax interfaces.
package inmem

430
dax/inmem/versionstore.go
View file

@ -1,430 +0,0 @@
package inmem
import (
"context"
"sort"
"sync"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/errors"
)
// Ensure type implements interface.
var _ dax.VersionStore = (*VersionStore)(nil)
// VersionStore manages all version info for shard, table keys, and field keys.
type VersionStore struct {
mu sync.RWMutex
// shards is a map of all shards, by table, by shard number, known to
// contain data.
shards map[dax.TableQualifierKey]map[dax.TableID]map[dax.ShardNum]dax.VersionedShard
// tableKeys is a map of all partitions, by table, by partition number,
// known to contain key data.
tableKeys map[dax.TableQualifierKey]map[dax.TableID]map[dax.PartitionNum]int
// fieldKeys is a map of all fields, by table, known to contain key data.
fieldKeys map[dax.TableQualifierKey]map[dax.TableID]map[dax.FieldName]int
}
// NewVersionStore returns a new instance of VersionStore with default values.
func NewVersionStore() *VersionStore {
return &VersionStore{
shards: make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.ShardNum]dax.VersionedShard),
tableKeys: make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.PartitionNum]int),
fieldKeys: make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.FieldName]int),
}
}
// AddTable adds a table to be managed by VersionStore.
func (s *VersionStore) AddTable(ctx context.Context, qtid dax.QualifiedTableID) error {
s.mu.Lock()
defer s.mu.Unlock()
// This check is clunky; three maps contain the table, but we only check for
// existence in one of them. It also seems weird to check all three, because
// if we get in a state where one of the maps doesn't contain a table that
// the other maps do contain, the state of the data is in question.
if _, found := s.shards[qtid.TableQualifier.Key()][qtid.ID]; found {
return dax.NewErrTableIDExists(qtid)
}
// Initialize the maps in case VersionStore wasn't created with NewVersionStore().
if s.shards == nil {
s.shards = make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.ShardNum]dax.VersionedShard)
}
if s.tableKeys == nil {
s.tableKeys = make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.PartitionNum]int)
}
if s.fieldKeys == nil {
s.fieldKeys = make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.FieldName]int)
}
// shards.
if _, ok := s.shards[qtid.TableQualifier.Key()]; !ok {
s.shards[qtid.TableQualifier.Key()] = make(map[dax.TableID]map[dax.ShardNum]dax.VersionedShard, 0)
}
if _, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]; !ok {
s.shards[qtid.TableQualifier.Key()][qtid.ID] = make(map[dax.ShardNum]dax.VersionedShard, 0)
}
// tableKeys.
if _, ok := s.tableKeys[qtid.TableQualifier.Key()]; !ok {
s.tableKeys[qtid.TableQualifier.Key()] = make(map[dax.TableID]map[dax.PartitionNum]int, 0)
}
if _, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]; !ok {
s.tableKeys[qtid.TableQualifier.Key()][qtid.ID] = make(map[dax.PartitionNum]int, 0)
}
// fieldKeys.
if _, ok := s.fieldKeys[qtid.TableQualifier.Key()]; !ok {
s.fieldKeys[qtid.TableQualifier.Key()] = make(map[dax.TableID]map[dax.FieldName]int, 0)
}
if _, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]; !ok {
s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID] = make(map[dax.FieldName]int, 0)
}
return nil
}
// RemoveTable removes the given table. An error will be returned if the table
// does not exist.
func (s *VersionStore) RemoveTable(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedShards, dax.VersionedPartitions, error) {
s.mu.Lock()
defer s.mu.Unlock()
var foundTable bool
var shards dax.VersionedShards
var partitions dax.VersionedPartitions
var err error
// Remove shards for table.
if s.shards != nil {
if _, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]; ok {
foundTable = true
// Get the shards to return before deleting from map.
shards, _, err = s.shardSlice(qtid)
if err != nil {
return nil, nil, errors.Wrapf(err, "getting shard slice: %s", qtid)
}
// Remove the shards.
delete(s.shards[qtid.TableQualifier.Key()], qtid.ID)
}
}
// Remove tableKeys for table.
if s.tableKeys != nil {
if _, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]; ok {
foundTable = true
// Get the partitions to return before deleting from map.
partitions, _, err = s.partitionSlice(qtid)
if err != nil {
return nil, nil, errors.Wrapf(err, "getting partition slice: %s", qtid)
}
// Remove the tableKeys.
delete(s.tableKeys[qtid.TableQualifier.Key()], qtid.ID)
}
}
// Remove fieldKeys for table.
if s.fieldKeys != nil {
if _, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]; ok {
foundTable = true
// Remove the fieldKeys.
delete(s.fieldKeys[qtid.TableQualifier.Key()], qtid.ID)
}
}
if !foundTable {
return nil, nil, dax.NewErrTableIDDoesNotExist(qtid)
}
return shards, partitions, nil
}
// AddShards adds new shards to be managed by VersionStore. It returns the
// number of shards added or an error.
func (s *VersionStore) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.VersionedShard) error {
s.mu.Lock()
defer s.mu.Unlock()
sh, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]
if !ok {
return dax.NewErrTableIDDoesNotExist(qtid)
}
var n int
for _, shard := range shards {
if _, ok := sh[shard.Num]; !ok {
n++ // TODO: this isn't considering a shard that exists, but the version changes.
}
sh[shard.Num] = shard
}
return nil
}
// Shards returns the list of shards available for the give table. It returns
// false if the table does not exist.
func (s *VersionStore) Shards(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedShards, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.shardSlice(qtid)
}
// shardSlice is an unprotected version of Shards().
func (s *VersionStore) shardSlice(qtid dax.QualifiedTableID) (dax.VersionedShards, bool, error) {
if s.shards == nil {
return nil, false, nil
}
if shardNumMap, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]; ok {
rtn := make(dax.VersionedShards, 0, len(shardNumMap))
for _, shard := range shardNumMap {
rtn = append(rtn, shard)
}
sort.Sort(rtn)
return rtn, true, nil
}
return nil, false, nil
}
// ShardVersion return the current version for the given table/shardNum.
// If a version is not being tracked, it returns a bool value of false.
func (s *VersionStore) ShardVersion(ctx context.Context, qtid dax.QualifiedTableID, shardNum dax.ShardNum) (int, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
t, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]
if !ok {
return -1, false, nil
}
v, ok := t[shardNum]
if !ok {
return -1, false, nil
}
return v.Version, true, nil
}
func (s *VersionStore) ShardTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) {
s.mu.RLock()
defer s.mu.RUnlock()
qual.Key()
tableIDs := make(dax.TableIDs, 0, len(s.shards[qual.Key()]))
for tableID := range s.shards[qual.Key()] {
tableIDs = append(tableIDs, tableID)
}
return tableIDs, nil
}
// AddPartitions adds new partitions to be managed by VersionStore. It returns
// the number of partitions added or an error.
func (s *VersionStore) AddPartitions(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.VersionedPartition) error {
s.mu.Lock()
defer s.mu.Unlock()
tk, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]
if !ok {
return dax.NewErrTableIDDoesNotExist(qtid)
}
for _, partition := range partitions {
tk[partition.Num] = partition.Version
}
return nil
}
// Partitions returns the list of partitions available for the give table. It
// returns false if the table does not exist.
func (s *VersionStore) Partitions(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedPartitions, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.partitionSlice(qtid)
}
// partitionSlice is an unprotected version of Partitions().
func (s *VersionStore) partitionSlice(qtid dax.QualifiedTableID) (dax.VersionedPartitions, bool, error) {
if s.tableKeys == nil {
return nil, false, nil
}
if partitionNumMap, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]; ok {
rtn := make(dax.VersionedPartitions, 0, len(partitionNumMap))
for partitionNum, version := range partitionNumMap {
rtn = append(rtn, dax.NewVersionedPartition(partitionNum, version))
}
sort.Sort(rtn)
return rtn, true, nil
}
return nil, false, nil
}
// PartitionVersion return the current version for the given table/partitionNum.
// If a version is not being tracked, it returns a bool value of false.
func (s *VersionStore) PartitionVersion(ctx context.Context, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) (int, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
t, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]
if !ok {
return -1, false, nil
}
v, ok := t[partitionNum]
if !ok {
return -1, false, nil
}
return v, true, nil
}
func (s *VersionStore) PartitionTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) {
s.mu.RLock()
defer s.mu.RUnlock()
tableIDs := make(dax.TableIDs, 0, len(s.tableKeys[qual.Key()]))
for tableName := range s.tableKeys[qual.Key()] {
tableIDs = append(tableIDs, tableName)
}
return tableIDs, nil
}
// AddFields adds new fields to be managed by VersionStore. It returns the
// number of fields added or an error.
func (s *VersionStore) AddFields(ctx context.Context, qtid dax.QualifiedTableID, fields ...dax.VersionedField) error {
s.mu.Lock()
defer s.mu.Unlock()
fk, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]
if !ok {
return dax.NewErrTableIDDoesNotExist(qtid)
}
for _, field := range fields {
fk[field.Name] = field.Version
}
return nil
}
// Fields returns the list of fields available for the give table. It returns
// false if the table does not exist.
func (s *VersionStore) Fields(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedFields, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.fieldSlice(qtid)
}
// fieldSlice is an unprotected version of Fields().
func (s *VersionStore) fieldSlice(qtid dax.QualifiedTableID) (dax.VersionedFields, bool, error) {
if s.fieldKeys == nil {
return nil, false, nil
}
if fieldNameMap, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]; ok {
rtn := make(dax.VersionedFields, 0, len(fieldNameMap))
for fieldName, version := range fieldNameMap {
rtn = append(rtn, dax.NewVersionedField(fieldName, version))
}
sort.Sort(rtn)
return rtn, true, nil
}
return nil, false, nil
}
// FieldVersion return the current version for the given table/field.
// If a version is not being tracked, it returns a bool value of false.
func (s *VersionStore) FieldVersion(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName) (int, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
t, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]
if !ok {
return -1, false, nil
}
v, ok := t[field]
if !ok {
return -1, false, nil
}
return v, true, nil
}
func (s *VersionStore) FieldTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) {
s.mu.RLock()
defer s.mu.RUnlock()
tableIDs := make(dax.TableIDs, 0, len(s.fieldKeys[qual.Key()]))
for tableID := range s.fieldKeys[qual.Key()] {
tableIDs = append(tableIDs, tableID)
}
return tableIDs, nil
}
// Copy returns a new copy of VersionStore.
func (s *VersionStore) Copy(ctx context.Context) (dax.VersionStore, error) {
s.mu.RLock()
defer s.mu.RUnlock()
new := NewVersionStore()
// shards.
for qkey, tableIDs := range s.shards {
for tableID, shards := range tableIDs {
qual := dax.NewTableQualifier(qkey.OrganizationID(), qkey.DatabaseID())
qtid := dax.NewQualifiedTableID(qual, tableID)
_ = new.AddTable(ctx, qtid)
for shardNum, shard := range shards {
new.shards[qual.Key()][tableID][shardNum] = shard
}
}
}
// tableKeys.
for qkey, tableIDs := range s.tableKeys {
for tableID, partitions := range tableIDs {
qual := dax.NewTableQualifier(qkey.OrganizationID(), qkey.DatabaseID())
qtid := dax.NewQualifiedTableID(qual, tableID)
_ = new.AddTable(ctx, qtid)
for partitionNum, version := range partitions {
new.tableKeys[qual.Key()][tableID][partitionNum] = version
}
}
}
// fieldKeys.
for qkey, tableIDs := range s.fieldKeys {
for tableID, fields := range tableIDs {
qual := dax.NewTableQualifier(qkey.OrganizationID(), qkey.DatabaseID())
qtid := dax.NewQualifiedTableID(qual, tableID)
_ = new.AddTable(ctx, qtid)
for field, version := range fields {
new.fieldKeys[qual.Key()][tableID][field] = version
}
}
}
return new, nil
}

166
dax/inmem/versionstore_test.go
View file

@ -1,166 +0,0 @@
package inmem_test
import (
"context"
"testing"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/inmem"
"github.com/featurebasedb/featurebase/v3/errors"
"github.com/stretchr/testify/assert"
)
func TestVersionStore(t *testing.T) {
orgID := dax.OrganizationID("acme")
dbID := dax.DatabaseID("db1")
tableID := dax.TableID("0000000000000001")
qual := dax.NewTableQualifier(orgID, dbID)
qtid := dax.NewQualifiedTableID(qual, tableID)
invalidQtid := dax.NewQualifiedTableID(qual, dax.TableID("0000000000000000"))
ctx := context.Background()
// Ensure that when using a Schemar not initiated with NewSchemar, the error
// handling works as expected.
t.Run("EmptyVersionStore", func(t *testing.T) {
s := inmem.VersionStore{}
t.Run("GetShardsInvalid", func(t *testing.T) {
sh, ok, err := s.Shards(ctx, invalidQtid)
assert.NoError(t, err)
assert.False(t, ok)
assert.Nil(t, sh)
})
// Add new table.
assert.NoError(t, s.AddTable(ctx, qtid))
})
t.Run("NewVersionStore", func(t *testing.T) {
s := inmem.NewVersionStore()
// Add new table.
assert.NoError(t, s.AddTable(ctx, qtid))
t.Run("AddTableAgain", func(t *testing.T) {
err := s.AddTable(ctx, qtid)
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrTableIDExists))
}
})
t.Run("AddShards", func(t *testing.T) {
err := s.AddShards(ctx, invalidQtid, dax.NewVersionedShard(1, 0))
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist))
}
{
_, ok, err := s.Shards(ctx, invalidQtid)
assert.NoError(t, err)
assert.False(t, ok)
}
// Shards is empty if no shards have been added.
{
sh, ok, err := s.Shards(ctx, qtid)
assert.NoError(t, err)
assert.True(t, ok)
assert.Equal(t, sh, dax.VersionedShards{})
}
// Add the first set of shards (with a duplicate (8)).
{
err := s.AddShards(ctx, qtid,
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(9, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(10, 0),
)
assert.NoError(t, err)
}
{
sh, ok, err := s.Shards(ctx, qtid)
assert.NoError(t, err)
assert.True(t, ok)
assert.Equal(t, dax.VersionedShards{
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(9, 0),
dax.NewVersionedShard(10, 0),
}, sh)
}
// Add another set of shards (with one duplicate (11) and one
// existing (10)).
{
err := s.AddShards(ctx, qtid,
dax.NewVersionedShard(10, 0),
dax.NewVersionedShard(11, 0),
dax.NewVersionedShard(12, 0),
dax.NewVersionedShard(11, 0),
)
assert.NoError(t, err)
}
{
sh, ok, err := s.Shards(ctx, qtid)
assert.NoError(t, err)
assert.True(t, ok)
assert.Equal(t, dax.VersionedShards{
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(9, 0),
dax.NewVersionedShard(10, 0),
dax.NewVersionedShard(11, 0),
dax.NewVersionedShard(12, 0),
}, sh)
}
})
t.Run("RemoveTable", func(t *testing.T) {
shards, partitions, err := s.RemoveTable(ctx, qtid)
assert.NoError(t, err)
assert.Equal(t, dax.VersionedPartitions{}, partitions)
assert.Equal(t, dax.VersionedShards{
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(9, 0),
dax.NewVersionedShard(10, 0),
dax.NewVersionedShard(11, 0),
dax.NewVersionedShard(12, 0),
}, shards)
// Make sure the table was removed.
shards, ok, err := s.Shards(ctx, qtid)
assert.NoError(t, err)
assert.False(t, ok)
assert.Nil(t, shards)
})
})
t.Run("ErrorConditions", func(t *testing.T) {
t.Run("JustSchemar", func(t *testing.T) {
s := inmem.VersionStore{}
shards, partitions, err := s.RemoveTable(ctx, qtid)
assert.Nil(t, shards)
assert.Nil(t, partitions)
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist))
}
})
t.Run("NewSchemar", func(t *testing.T) {
s := inmem.NewVersionStore()
shards, partitions, err := s.RemoveTable(ctx, qtid)
assert.Nil(t, shards)
assert.Nil(t, partitions)
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist))
}
})
})
}

9
dax/mds/controller/balancer.go
View file

@ -28,6 +28,11 @@ type Balancer interface {
// which are not assigned to any worker, that means the query
// would return incomplete data, so we want to error.
WorkersForJobPrefix(ctx context.Context, prefix string) ([]dax.WorkerInfo, error)
// RemoveJobs is for e.g. when dropping a table remove all jobs
// associated with that table without needing to look up in
// advance which shards or partitions are actually present.
RemoveJobs(ctx context.Context, prefix string) ([]dax.WorkerDiff, error)
}
// Ensure type implements interface.
@ -67,3 +72,7 @@ func (b *NopBalancer) WorkersForJobs(ctx context.Context, jobs []dax.Job) ([]dax
func (b *NopBalancer) WorkersForJobPrefix(ctx context.Context, prefix string) ([]dax.WorkerInfo, error) {
return []dax.WorkerInfo{}, nil
}
func (b *NopBalancer) RemoveJobs(ctx context.Context, prefix string) ([]dax.WorkerDiff, error) {
return nil, nil
}

374
dax/mds/controller/controller.go
View file

@ -22,9 +22,6 @@ type Controller struct {
// initialization.
mu sync.RWMutex
// versionStore
versionStore dax.VersionStore
// Schemar used by the controller to get table information. The controller
// should NOT call Schemar methods which modify data. Schema mutations are
// made outside of the controller (at this point that happens in MDS).
@ -84,11 +81,6 @@ func New(cfg Config) *Controller {
switch cfg.StorageMethod {
case "boltdb":
if err := cfg.BoltDB.InitializeBuckets(boltdb.VersionStoreBuckets...); err != nil {
c.logger.Panicf("initializing version store buckets: %v", err)
}
c.versionStore = boltdb.NewVersionStore(cfg.BoltDB, c.logger)
if err := cfg.BoltDB.InitializeBuckets(boltdb.NodeServiceBuckets...); err != nil {
c.logger.Panicf("initializing node service buckets: %v", err)
}
@ -520,12 +512,6 @@ func (c *Controller) nodesTranslateReadOrWrite(ctx context.Context, role *dax.Tr
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
// Initialize the partition version to 0.
qtid := j.table().QualifiedTableID()
if err := c.versionStore.AddPartitions(ctx, qtid, dax.NewVersionedPartition(j.partitionNum(), 0)); err != nil {
return nil, false, NewErrInternal(err.Error())
}
}
// Convert the slice of addresses into a slice of addressMethod containing
@ -546,26 +532,14 @@ func (c *Controller) nodesTranslateReadOrWrite(ctx context.Context, role *dax.Tr
for _, worker := range workers {
// covert worker.Jobs []string to map[string][]Partition
translateMap := make(map[dax.TableKey]dax.VersionedPartitions)
translateMap := make(map[dax.TableKey]dax.PartitionNums)
for _, job := range worker.Jobs {
j, err := decodePartition(job)
if err != nil {
return nil, false, NewErrInternal(err.Error())
}
// Get the partition version from the local versionStore.
tkey := j.table()
qtid := tkey.QualifiedTableID()
partitionVersion, found, err := c.versionStore.PartitionVersion(ctx, qtid, j.partitionNum())
if err != nil {
return nil, false, err
} else if !found {
return nil, false, NewErrInternal("partition version not found in cache")
}
translateMap[tkey] = append(translateMap[tkey],
dax.NewVersionedPartition(j.partitionNum(), partitionVersion),
)
translateMap[j.table()] = append(translateMap[j.table()], j.partitionNum())
}
for table, partitions := range translateMap {
@ -707,12 +681,6 @@ func (c *Controller) nodesComputeReadOrWrite(ctx context.Context, role *dax.Comp
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
// Initialize the shard version to 0.
qtid := j.table().QualifiedTableID()
if err := c.versionStore.AddShards(ctx, qtid, dax.NewVersionedShard(j.shardNum(), 0)); err != nil {
return nil, false, NewErrInternal(err.Error())
}
}
// Convert the slice of addresses into a slice of addressMethod containing
@ -739,26 +707,14 @@ func (c *Controller) workersToAssignedNodes(ctx context.Context, workers []dax.W
nodes := []dax.AssignedNode{}
for _, worker := range workers {
// convert worker.Jobs []string to map[TableName][]Shard
computeMap := make(map[dax.TableKey]dax.VersionedShards)
computeMap := make(map[dax.TableKey]dax.ShardNums)
for _, job := range worker.Jobs {
j, err := decodeShard(job)
if err != nil {
return nil, NewErrInternal(err.Error())
}
// Get the shard version from the local versionStore.
tkey := j.table()
qtid := tkey.QualifiedTableID()
shardVersion, found, err := c.versionStore.ShardVersion(ctx, qtid, j.shardNum())
if err != nil {
return nil, err
} else if !found {
return nil, NewErrInternal("shard version not found in cache")
}
computeMap[tkey] = append(computeMap[tkey],
dax.NewVersionedShard(j.shardNum(), shardVersion),
)
computeMap[j.table()] = append(computeMap[j.table()], j.shardNum())
}
for table, shards := range computeMap {
@ -783,31 +739,6 @@ func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable)
c.mu.Lock()
defer c.mu.Unlock()
qtid := qtbl.QualifiedID()
// Add the table to the versionStore.
if err := c.versionStore.AddTable(ctx, qtid); err != nil {
return errors.Wrapf(err, "adding table: %s", qtid)
}
// Add fields which have string keys to the local versionStore.
fieldVersions := make(dax.VersionedFields, 0)
for _, field := range qtbl.Fields {
if !field.StringKeys() {
continue
}
fieldVersions = append(fieldVersions, dax.VersionedField{
Name: field.Name,
Version: 0,
})
}
if len(fieldVersions) > 0 {
if err := c.versionStore.AddFields(ctx, qtid, fieldVersions...); err != nil {
return errors.Wrapf(err, "adding fields: %s, %v", qtid, fieldVersions)
}
}
// If the table is keyed, add partitions to the balancer.
if qtbl.StringKeys() {
// workerSet maintains the set of workers which have a job assignment change
@ -815,15 +746,9 @@ func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable)
workerSet := NewAddressSet()
// Generate the list of partitionsToAdd to be added.
partitionsToAdd := make(dax.VersionedPartitions, qtbl.PartitionN)
partitionsToAdd := make(dax.PartitionNums, qtbl.PartitionN)
for partitionNum := 0; partitionNum < qtbl.PartitionN; partitionNum++ {
partitionsToAdd[partitionNum] = dax.NewVersionedPartition(dax.PartitionNum(partitionNum), 0)
}
// Add partitions to versionStore. Version is intentionally set to 0
// here as this is the initial instance of the partition.
if err := c.versionStore.AddPartitions(ctx, qtid, partitionsToAdd...); err != nil {
return NewErrInternal(err.Error())
partitionsToAdd[partitionNum] = dax.PartitionNum(partitionNum)
}
stringers := make([]fmt.Stringer, 0, len(partitionsToAdd))
@ -858,13 +783,7 @@ func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable)
// and therefore need to be sent an updated Directive.
workerSet := NewAddressSet()
p := dax.NewVersionedPartition(0, 0)
// Add partition 0 to versionStore. Version is intentionally set to 0
// here as this is the initial instance of the partition.
if err := c.versionStore.AddPartitions(ctx, qtid, p); err != nil {
return NewErrInternal(err.Error())
}
p := dax.PartitionNum(0)
// We don't currently use the returned diff, other than to determine
// which worker was affected, because we send the full Directive
@ -900,39 +819,24 @@ func (c *Controller) DropTable(ctx context.Context, qtid dax.QualifiedTableID) e
return errors.Wrapf(err, "table not in schemar: %s", qtid)
}
// Remove the table from the versionStore.
// Since the schemar should be the system of record for the existence of a
// table, if the versionStore is not aware of the table, we just log it and
// continue.
shards, partitions, err := c.versionStore.RemoveTable(ctx, qtid)
if err != nil {
return errors.Wrapf(err, "removing table: %s", qtid)
}
// workerSet maintains the set of workers which have a job assignment change
// and therefore need to be sent an updated Directive.
workerSet := NewAddressSet()
// Remove shards.
for _, s := range shards {
diffs, err := c.ComputeBalancer.RemoveJob(ctx, shard(qtid.Key(), s))
if err != nil {
return errors.Wrap(err, "removing job")
}
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
diffs, err := c.ComputeBalancer.RemoveJobs(ctx, string(qtid.Key()))
if err != nil {
return errors.Wrap(err, "removing jobs")
}
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
// Remove partitions.
for _, p := range partitions {
diffs, err := c.TranslateBalancer.RemoveJob(ctx, partition(qtid.Key(), p))
if err != nil {
return errors.Wrap(err, "removing job")
}
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
diffs, err = c.TranslateBalancer.RemoveJobs(ctx, string(qtid.Key()))
if err != nil {
return errors.Wrap(err, "removing job")
}
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
// Convert the slice of addresses into a slice of addressMethod containing
@ -966,15 +870,10 @@ func (c *Controller) Tables(ctx context.Context, qual dax.TableQualifier, ids ..
// AddShards registers the table/shard combinations with the controller and
// sends the necessary directive.
func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.VersionedShard) error {
func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) error {
c.mu.Lock()
defer c.mu.Unlock()
// Add shards to versionStore.
if err := c.versionStore.AddShards(ctx, qtid, shards...); err != nil {
return errors.Wrapf(err, "adding shards: %s, %v", qtid, shards)
}
// workerSet maintains the set of workers which have a job assignment change
// and therefore need to be sent an updated Directive.
workerSet := NewAddressSet()
@ -990,11 +889,6 @@ func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, s
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
// Initialize the shard version to 0.
if err := c.versionStore.AddShards(ctx, qtid, dax.NewVersionedShard(s.Num, 0)); err != nil {
return NewErrInternal(err.Error())
}
}
// Convert the slice of addresses into a slice of addressMethod containing
@ -1010,7 +904,7 @@ func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, s
// RemoveShards deregisters the table/shard combinations with the controller and
// sends the necessary directives.
func (c *Controller) RemoveShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.VersionedShard) error {
func (c *Controller) RemoveShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) error {
c.mu.Lock()
defer c.mu.Unlock()
@ -1050,7 +944,7 @@ func (c *Controller) sendDirectives(ctx context.Context, addrs ...addressMethod)
return nil
}
directives, err := c.buildDirectives(ctx, addrs, c.versionStore)
directives, err := c.buildDirectives(ctx, addrs)
if err != nil {
return errors.Wrap(err, "building directives")
}
@ -1120,7 +1014,7 @@ func applyAddressMethod(addrs []dax.Address, method dax.DirectiveMethod) []addre
// buildDirectives builds a list of directives for the given addrs (i.e. nodes)
// using information (i.e. current state) from the balancers.
func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod, versionStore dax.VersionStore) ([]*dax.Directive, error) {
func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod) ([]*dax.Directive, error) {
directives := make([]*dax.Directive, len(addrs))
for i, addressMethod := range addrs {
@ -1141,12 +1035,12 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
// computeMap maps a table to a list of shards for that table. We need
// to aggregate them here because the list of jobs from WorkerState()
// can contain a mixture of table/shards.
computeMap := make(map[dax.TableKey][]dax.VersionedShard)
computeMap := make(map[dax.TableKey][]dax.ShardNum)
// translateMap maps a table to a list of partitions for that table. We
// need to aggregate them here because the list of jobs from
// WorkerState() can contain a mixture of table/partitions.
translateMap := make(map[dax.TableKey]dax.VersionedPartitions)
translateMap := make(map[dax.TableKey][]dax.PartitionNum)
// tableSet maintains the set of tables which have a job assignment
// change and therefore need to be included in the Directive schema.
@ -1174,24 +1068,8 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
return nil, errors.Wrapf(err, "decoding shard job: %s", job)
}
// The Shard object decoded from the balancer doesn't
// contain a valid version (because the balancer
// intentionally does not store version information). Here,
// we get the current shard version from the controller's
// cache (i.e. versionStore) and inject that into the Shard
// sent in the directive.
tkey := j.table()
qtid := tkey.QualifiedTableID()
shardVersion, found, err := versionStore.ShardVersion(ctx, qtid, j.shardNum())
if err != nil {
return nil, errors.Wrapf(err, "getting shard version: %s, %d", qtid, j.shardNum())
} else if !found {
return nil, NewErrInternal("shard version not found in cache")
}
computeMap[tkey] = append(computeMap[tkey],
dax.NewVersionedShard(j.shardNum(), shardVersion),
)
computeMap[tkey] = append(computeMap[tkey], j.shardNum())
tableSet.Add(tkey)
}
case dax.RoleTypeTranslate:
@ -1209,24 +1087,8 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
ownsPartition0[j.table()] = struct{}{}
}
// The Partition object decoded from the balancer doesn't
// contain a valid version (because the balancer
// intentionally does not store version information). Here,
// we get the current partition version from the
// controller's cache (i.e. versionStore) and inject that
// into the Partition sent in the directive.
tkey := j.table()
qtid := tkey.QualifiedTableID()
partitionVersion, found, err := versionStore.PartitionVersion(ctx, qtid, j.partitionNum())
if err != nil {
return nil, errors.Wrapf(err, "getting partition version: %s, %d", qtid, j.partitionNum())
} else if !found {
return nil, NewErrInternal("partition version not found in cache")
}
translateMap[tkey] = append(translateMap[tkey],
dax.NewVersionedPartition(j.partitionNum(), partitionVersion),
)
translateMap[tkey] = append(translateMap[tkey], j.partitionNum())
tableSet.Add(tkey)
}
}
@ -1237,7 +1099,7 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
// Because these were encoded as strings in the balancer and may be
// out of order numerically, sort them as integers.
//sort.Slice(v, func(i, j int) bool { return v[i] < v[j] })
sort.Sort(dax.VersionedShards(v))
sort.Sort(dax.ShardNums(v))
d.ComputeRoles = append(d.ComputeRoles, dax.ComputeRole{
TableKey: k,
@ -1249,7 +1111,7 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
for k, v := range translateMap {
// Because these were encoded as strings in the balancer and may be
// out of order numerically, sort them as integers.
sort.Sort(v)
sort.Sort(dax.PartitionNums(v))
d.TranslateRoles = append(d.TranslateRoles, dax.TranslateRole{
TableKey: k,
@ -1271,7 +1133,7 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
return nil, errors.Wrapf(err, "getting table: %s", tkey)
}
fieldVersions := make(dax.VersionedFields, 0)
fieldNames := make([]dax.FieldName, 0)
for _, field := range table.Fields {
if !field.StringKeys() {
continue
@ -1282,26 +1144,16 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
continue
}
fieldVersion, found, err := versionStore.FieldVersion(ctx, qtid, field.Name)
if err != nil {
return nil, errors.Wrapf(err, "getting field version: %s, %s", qtid, field)
} else if !found {
return nil, NewErrInternal("field version not found in cache")
}
fieldVersions = append(fieldVersions, dax.VersionedField{
Name: field.Name,
Version: fieldVersion,
})
fieldNames = append(fieldNames, field.Name)
}
if len(fieldVersions) == 0 {
if len(fieldNames) == 0 {
continue
}
d.TranslateRoles = append(d.TranslateRoles, dax.TranslateRole{
TableKey: tkey,
Fields: fieldVersions,
Fields: fieldNames,
})
tableSet.Add(tkey)
@ -1355,48 +1207,50 @@ func (c *Controller) InitializePoller(ctx context.Context) error {
}
// SnapshotTable snapshots a table.
// TODO(jaffee): do we need to re-implement this w/o version store? Or can we just remove entirely?
func (c *Controller) SnapshotTable(ctx context.Context, qtid dax.QualifiedTableID) error {
shards, ok, err := c.versionStore.Shards(ctx, qtid)
if err != nil {
return errors.Wrap(err, "getting shards from version store")
} else if !ok {
return errors.New(errors.ErrUncoded, "got false back from versionStore.Shards")
}
for _, shard := range shards {
if err := c.SnapshotShardData(ctx, qtid, shard.Num); err != nil {
return errors.Wrapf(err, "snapshotting shard data: qtid: %s, shard: %d", qtid, shard.Num)
}
}
partitions, ok, err := c.versionStore.Partitions(ctx, qtid)
if err != nil {
return errors.Wrap(err, "getting partitions from version store")
} else if !ok {
return errors.New(errors.ErrUncoded, "got false back from versionStore.Partitions")
}
for _, part := range partitions {
if err := c.SnapshotTableKeys(ctx, qtid, part.Num); err != nil {
return errors.Wrapf(err, "snapshotting table keys: qtid: %s, partition: %d", qtid, part.Num)
}
}
fields, ok, err := c.versionStore.Fields(ctx, qtid)
if err != nil {
return errors.Wrap(err, "getting fields from version store")
} else if !ok {
return errors.New(errors.ErrUncoded, "got false back from versionStore.Fields")
}
for _, fld := range fields {
if fld.Name != "_id" {
if err := c.SnapshotFieldKeys(ctx, qtid, fld.Name); err != nil {
return errors.Wrapf(err, "snapshotting field keys: qtid: %s, field: %s", qtid, fld.Name)
}
}
}
return nil
// shards, ok, err := c.versionStore.Shards(ctx, qtid)
// if err != nil {
// return errors.Wrap(err, "getting shards from version store")
// } else if !ok {
// return errors.New(errors.ErrUncoded, "got false back from versionStore.Shards")
// }
// for _, shard := range shards {
// if err := c.SnapshotShardData(ctx, qtid, shard.Num); err != nil {
// return errors.Wrapf(err, "snapshotting shard data: qtid: %s, shard: %d", qtid, shard.Num)
// }
// }
// partitions, ok, err := c.versionStore.Partitions(ctx, qtid)
// if err != nil {
// return errors.Wrap(err, "getting partitions from version store")
// } else if !ok {
// return errors.New(errors.ErrUncoded, "got false back from versionStore.Partitions")
// }
// for _, part := range partitions {
// if err := c.SnapshotTableKeys(ctx, qtid, part.Num); err != nil {
// return errors.Wrapf(err, "snapshotting table keys: qtid: %s, partition: %d", qtid, part.Num)
// }
// }
// fields, ok, err := c.versionStore.Fields(ctx, qtid)
// if err != nil {
// return errors.Wrap(err, "getting fields from version store")
// } else if !ok {
// return errors.New(errors.ErrUncoded, "got false back from versionStore.Fields")
// }
// for _, fld := range fields {
// if fld.Name != "_id" {
// if err := c.SnapshotFieldKeys(ctx, qtid, fld.Name); err != nil {
// return errors.Wrapf(err, "snapshotting field keys: qtid: %s, field: %s", qtid, fld.Name)
// }
// }
// }
// return nil
}
// SnapshotShardData forces the compute node responsible for the given shard to
@ -1405,7 +1259,7 @@ func (c *Controller) SnapshotTable(ctx context.Context, qtid dax.QualifiedTableI
func (c *Controller) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTableID, shardNum dax.ShardNum) error {
// Get the node responsible for the shard.
bal := c.ComputeBalancer
job := shard(qtid.Key(), dax.NewVersionedShard(shardNum, -1))
job := shard(qtid.Key(), shardNum)
workers, err := bal.WorkersForJobs(ctx, []dax.Job{dax.Job(job.String())})
if err != nil {
return errors.Wrapf(err, "getting workers for jobs: %s", job)
@ -1437,7 +1291,7 @@ func (c *Controller) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTa
func (c *Controller) SnapshotTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) error {
// Get the node responsible for the partition.
bal := c.TranslateBalancer
job := partition(qtid.Key(), dax.NewVersionedPartition(partitionNum, -1))
job := partition(qtid.Key(), partitionNum)
workers, err := bal.WorkersForJobs(ctx, []dax.Job{dax.Job(job.String())})
if err != nil {
return errors.Wrapf(err, "getting workers for jobs: %s", job)
@ -1471,7 +1325,7 @@ func (c *Controller) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTa
bal := c.TranslateBalancer
// Field translation is currently handled by partition 0.
partitionNum := dax.PartitionNum(0)
job := partition(qtid.Key(), dax.NewVersionedPartition(partitionNum, -1))
job := partition(qtid.Key(), partitionNum)
workers, err := bal.WorkersForJobs(ctx, []dax.Job{dax.Job(job.String())})
if err != nil {
@ -1503,7 +1357,7 @@ func (c *Controller) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTa
func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards dax.ShardNums, isWrite bool) ([]dax.ComputeNode, error) {
inRole := &dax.ComputeRole{
TableKey: qtid.Key(),
Shards: dax.NewVersionedShards(shards...),
Shards: shards,
}
nodes, err := c.Nodes(ctx, inRole, isWrite)
@ -1524,7 +1378,7 @@ func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID
computeNodes = append(computeNodes, dax.ComputeNode{
Address: node.Address,
Table: role.TableKey,
Shards: role.Shards.Nums(),
Shards: role.Shards,
})
}
@ -1534,7 +1388,7 @@ func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID
func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions dax.PartitionNums, isWrite bool) ([]dax.TranslateNode, error) {
inRole := &dax.TranslateRole{
TableKey: qtid.Key(),
Partitions: dax.NewVersionedPartitions(partitions...),
Partitions: partitions,
}
nodes, err := c.Nodes(ctx, inRole, isWrite)
@ -1555,7 +1409,7 @@ func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTable
translateNodes = append(translateNodes, dax.TranslateNode{
Address: node.Address,
Table: role.TableKey,
Partitions: role.Partitions.Nums(),
Partitions: role.Partitions,
})
}
@ -1612,37 +1466,19 @@ func (c *Controller) CreateField(ctx context.Context, qtid dax.QualifiedTableID,
// and therefore need to be sent an updated Directive.
workerSet := NewAddressSet()
// If the field has string keys, add it to the local versionStore.
if fld.StringKeys() {
fieldVersion := dax.VersionedField{
Name: fld.Name,
Version: 0,
}
if err := c.versionStore.AddFields(ctx, qtid, fieldVersion); err != nil {
return errors.Wrapf(err, "adding fields: %s, %s", qtid, fieldVersion)
}
// Get the worker(s) responsible for partition 0.
job := partition(qtid.Key(), 0).String()
workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)})
if err != nil {
return errors.Wrapf(err, "getting workers for job: %s", job)
}
// Get the worker responsible for partition 0, which handles field key
// translation. Be sure to get the current version.
if v, found, err := c.versionStore.PartitionVersion(ctx, qtid, 0); err != nil {
return errors.Wrapf(err, "getting partition version: %s/0", qtid)
} else if found {
// Get the worker(s) responsible for partition 0.
job := partition(qtid.Key(), dax.VersionedPartition{
Num: 0,
Version: v,
}).String()
workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)})
if err != nil {
return errors.Wrapf(err, "getting workers for job: %s", job)
}
for _, w := range workers {
workerSet.Add(dax.Address(w.ID))
}
for _, w := range workers {
workerSet.Add(dax.Address(w.ID))
}
// TODO if there isn't already a worker for partition 0, do we need to add it?
// Get the list of workers responsible for shard data for this table.
if state, err := c.ComputeBalancer.CurrentState(ctx); err != nil {
return errors.Wrap(err, "getting current compute state")
@ -1679,27 +1515,15 @@ func (c *Controller) DropField(ctx context.Context, qtid dax.QualifiedTableID, f
// and therefore need to be sent an updated Directive.
workerSet := NewAddressSet()
// If the field has string keys, remove it from the local versionStore.
// TODO: implement RemoveField() on VersionStore interface.
// Get the worker(s) responsible for partition 0.
job := partition(qtid.Key(), 0).String()
workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)})
if err != nil {
return errors.Wrapf(err, "getting workers for job: %s", job)
}
// Get the worker responsible for partition 0, which handles field key
// translation. Be sure to get the current version.
if v, found, err := c.versionStore.PartitionVersion(ctx, qtid, 0); err != nil {
return errors.Wrapf(err, "getting partition version: %s/0", qtid)
} else if found {
// Get the worker(s) responsible for partition 0.
job := partition(qtid.Key(), dax.VersionedPartition{
Num: 0,
Version: v,
}).String()
workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)})
if err != nil {
return errors.Wrapf(err, "getting workers for job: %s", job)
}
for _, w := range workers {
workerSet.Add(dax.Address(w.ID))
}
for _, w := range workers {
workerSet.Add(dax.Address(w.ID))
}
// Get the list of workers responsible for shard data for this table.

449
dax/mds/controller/controller_test.go
View file

@ -109,18 +109,17 @@ func TestController(t *testing.T) {
assert.Equal(t, exp, director.flush())
// Add the same non-keyed table again.
err := con.CreateTable(ctx, tbl0)
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrTableIDExists))
}
// TODO(jaffee) figure out what this should do
// err := con.CreateTable(ctx, tbl0)
// if assert.Error(t, err) {
// assert.True(t, errors.Is(err, dax.ErrTableIDExists))
// }
exp = []*dax.Directive{}
assert.Equal(t, exp, director.flush())
// Add a shard.
assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(),
dax.NewVersionedShard(0, 0),
))
assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), 0))
exp = []*dax.Directive{
{
@ -132,9 +131,7 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
},
Shards: dax.NewShardNums(0),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -185,13 +182,7 @@ func TestController(t *testing.T) {
assert.Equal(t, exp, director.flush())
// Add more shards.
assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
))
assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), dax.NewShardNums(1, 2, 3, 5, 8)...))
exp = []*dax.Directive{
{
@ -203,10 +194,7 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(3, 0),
},
Shards: dax.NewShardNums(0, 3),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -221,10 +209,7 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(5, 0),
},
Shards: dax.NewShardNums(1, 5),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -239,10 +224,7 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(2, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -257,12 +239,7 @@ func TestController(t *testing.T) {
assert.NoError(t, con.CreateTable(ctx, tbl1))
// Add more shards.
assert.NoError(t, con.AddShards(ctx, tbl1.QualifiedID(),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(13, 0),
))
assert.NoError(t, con.AddShards(ctx, tbl1.QualifiedID(), dax.NewShardNums(3, 5, 8, 13)...))
exp = []*dax.Directive{
{
@ -275,17 +252,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 13),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(3, 0),
},
Shards: dax.NewShardNums(0, 3),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -301,16 +272,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(5, 0),
},
Shards: dax.NewShardNums(5),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(5, 0),
},
Shards: dax.NewShardNums(1, 5),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -326,16 +292,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(8),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(2, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -358,18 +319,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 13),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(3, 0),
},
Shards: dax.NewShardNums(0, 1, 3),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -385,18 +339,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(5, 8),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(2, 5, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -419,23 +366,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 5, 8, 13),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(0, 1, 2, 3, 5, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -470,23 +405,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 5, 8, 13),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(0, 1, 2, 3, 5, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -496,10 +419,7 @@ func TestController(t *testing.T) {
assert.Equal(t, exp, director.flush())
// Remove shards.
assert.NoError(t, con.RemoveShards(ctx, tbl0.QualifiedID(),
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(5, 0),
))
assert.NoError(t, con.RemoveShards(ctx, tbl0.QualifiedID(), dax.NewShardNums(2, 5)...))
exp = []*dax.Directive{
{
@ -512,21 +432,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 5, 8, 13),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(0, 1, 3, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -538,10 +448,7 @@ func TestController(t *testing.T) {
// Remove shards, one which does not exist.
// Currently that doesn't result in an error, it simply no-ops on trying
// to remove 99.
assert.NoError(t, con.RemoveShards(ctx, tbl0.QualifiedID(),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(99, 0),
))
assert.NoError(t, con.RemoveShards(ctx, tbl0.QualifiedID(), dax.NewShardNums(3, 99)...))
exp = []*dax.Directive{
{
@ -554,20 +461,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 5, 8, 13),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(0, 1, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -589,12 +487,7 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 5, 8, 13),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -604,7 +497,7 @@ func TestController(t *testing.T) {
assert.Equal(t, exp, director.flush())
// Remove a node which doesn't exist.
err = con.DeregisterNodes(ctx, "invalidNode")
err := con.DeregisterNodes(ctx, "invalidNode")
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrNodeDoesNotExist))
}
@ -683,17 +576,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(2, 0),
dax.NewVersionedPartition(3, 0),
dax.NewVersionedPartition(4, 0),
dax.NewVersionedPartition(5, 0),
dax.NewVersionedPartition(6, 0),
dax.NewVersionedPartition(7, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0, 1, 2, 3, 4, 5, 6, 7),
},
},
Version: 2,
@ -720,13 +604,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(2, 0),
dax.NewVersionedPartition(3, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0, 1, 2, 3),
},
},
Version: 3,
@ -740,13 +619,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(4, 0),
dax.NewVersionedPartition(5, 0),
dax.NewVersionedPartition(6, 0),
dax.NewVersionedPartition(7, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(4, 5, 6, 7),
},
},
Version: 4,
@ -772,12 +646,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(2, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0, 1, 2),
},
},
Version: 5,
@ -791,12 +661,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(4, 0),
dax.NewVersionedPartition(5, 0),
dax.NewVersionedPartition(6, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(4, 5, 6),
},
},
Version: 6,
@ -810,11 +676,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(3, 0),
dax.NewVersionedPartition(7, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(3, 7),
},
},
Version: 7,
@ -842,25 +705,12 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl1.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(4, 0),
dax.NewVersionedPartition(7, 0),
dax.NewVersionedPartition(10, 0),
dax.NewVersionedPartition(13, 0),
dax.NewVersionedPartition(16, 0),
dax.NewVersionedPartition(19, 0),
dax.NewVersionedPartition(22, 0),
},
TableKey: tbl1.Key(),
Partitions: dax.NewPartitionNums(1, 4, 7, 10, 13, 16, 19, 22),
},
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(2, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0, 1, 2),
},
},
Version: 8,
@ -875,25 +725,12 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl1.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(2, 0),
dax.NewVersionedPartition(5, 0),
dax.NewVersionedPartition(8, 0),
dax.NewVersionedPartition(11, 0),
dax.NewVersionedPartition(14, 0),
dax.NewVersionedPartition(17, 0),
dax.NewVersionedPartition(20, 0),
dax.NewVersionedPartition(23, 0),
},
TableKey: tbl1.Key(),
Partitions: dax.NewPartitionNums(2, 5, 8, 11, 14, 17, 20, 23),
},
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(4, 0),
dax.NewVersionedPartition(5, 0),
dax.NewVersionedPartition(6, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(4, 5, 6),
},
},
Version: 9,
@ -908,24 +745,12 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl1.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(3, 0),
dax.NewVersionedPartition(6, 0),
dax.NewVersionedPartition(9, 0),
dax.NewVersionedPartition(12, 0),
dax.NewVersionedPartition(15, 0),
dax.NewVersionedPartition(18, 0),
dax.NewVersionedPartition(21, 0),
},
TableKey: tbl1.Key(),
Partitions: dax.NewPartitionNums(0, 3, 6, 9, 12, 15, 18, 21),
},
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(3, 0),
dax.NewVersionedPartition(7, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(3, 7),
},
},
Version: 10,
@ -947,17 +772,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl1.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(4, 0),
dax.NewVersionedPartition(7, 0),
dax.NewVersionedPartition(10, 0),
dax.NewVersionedPartition(13, 0),
dax.NewVersionedPartition(16, 0),
dax.NewVersionedPartition(19, 0),
dax.NewVersionedPartition(22, 0),
},
TableKey: tbl1.Key(),
Partitions: dax.NewPartitionNums(1, 4, 7, 10, 13, 16, 19, 22),
},
},
Version: 11,
@ -971,17 +787,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl1.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(2, 0),
dax.NewVersionedPartition(5, 0),
dax.NewVersionedPartition(8, 0),
dax.NewVersionedPartition(11, 0),
dax.NewVersionedPartition(14, 0),
dax.NewVersionedPartition(17, 0),
dax.NewVersionedPartition(20, 0),
dax.NewVersionedPartition(23, 0),
},
TableKey: tbl1.Key(),
Partitions: dax.NewPartitionNums(2, 5, 8, 11, 14, 17, 20, 23),
},
},
Version: 12,
@ -995,17 +802,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl1.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(3, 0),
dax.NewVersionedPartition(6, 0),
dax.NewVersionedPartition(9, 0),
dax.NewVersionedPartition(12, 0),
dax.NewVersionedPartition(15, 0),
dax.NewVersionedPartition(18, 0),
dax.NewVersionedPartition(21, 0),
},
TableKey: tbl1.Key(),
Partitions: dax.NewPartitionNums(0, 3, 6, 9, 12, 15, 18, 21),
},
},
Version: 13,
@ -1020,13 +818,11 @@ func TestController(t *testing.T) {
}
// Add shards to a table which doesn't exist.
err = con.AddShards(ctx, invalidQtid,
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(2, 0),
)
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist))
}
// TODO(jaffee) figure out what this should do
// err = con.AddShards(ctx, invalidQtid, dax.NewShardNums(1, 2)...)
// if assert.Error(t, err) {
// assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist))
// }
// Register an invalid node.
nodeX := &dax.Node{
@ -1082,14 +878,7 @@ func TestController(t *testing.T) {
assert.NoError(t, con.CreateTable(ctx, tbl0))
// Add shards.
assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(),
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(11, 0),
dax.NewVersionedShard(12, 0),
))
assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), 0, 1, 2, 3, 11, 12))
t.Run("ComputeRole", func(t *testing.T) {
tests := []struct {
@ -1100,27 +889,21 @@ func TestController(t *testing.T) {
{
role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.NewVersionedShards(0, 1, 2, 3),
Shards: dax.NewShardNums(0, 1, 2, 3),
},
exp: []dax.AssignedNode{
{
Address: node0.Address,
Role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(2, 0),
},
Shards: dax.NewShardNums(0, 2),
},
},
{
Address: node1.Address,
Role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(3, 0),
},
Shards: dax.NewShardNums(1, 3),
},
},
},
@ -1128,16 +911,14 @@ func TestController(t *testing.T) {
{
role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.NewVersionedShards(1),
Shards: dax.NewShardNums(1),
},
exp: []dax.AssignedNode{
{
Address: node1.Address,
Role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(1, 0),
},
Shards: dax.NewShardNums(1),
},
},
},
@ -1146,7 +927,7 @@ func TestController(t *testing.T) {
// Add unassigned shards.
role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.NewVersionedShards(1, 888, 889),
Shards: dax.NewShardNums(1, 888, 889),
},
isWrite: true,
exp: []dax.AssignedNode{
@ -1154,19 +935,14 @@ func TestController(t *testing.T) {
Address: node0.Address,
Role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(888, 0),
},
Shards: dax.NewShardNums(888),
},
},
{
Address: node1.Address,
Role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(889, 0),
},
Shards: dax.NewShardNums(1, 889),
},
},
},
@ -1175,17 +951,14 @@ func TestController(t *testing.T) {
// Ensure shards are not returned sorted as strings.
role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.NewVersionedShards(2, 11),
Shards: dax.NewShardNums(2, 11),
},
exp: []dax.AssignedNode{
{
Address: node0.Address,
Role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(11, 0),
},
Shards: dax.NewShardNums(2, 11),
},
},
},
@ -1209,75 +982,55 @@ func TestController(t *testing.T) {
}{
{
role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, -1),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0),
},
isWrite: true,
exp: []dax.AssignedNode{
{
Address: node0.Address,
Role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0),
},
},
},
},
{
role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, -1),
dax.NewVersionedPartition(1, -1),
dax.NewVersionedPartition(2, -1),
dax.NewVersionedPartition(3, -1),
dax.NewVersionedPartition(999, -1),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0, 1, 2, 3, 999),
},
isWrite: false,
exp: []dax.AssignedNode{
{
Address: node0.Address,
Role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(2, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0, 2),
},
},
{
Address: node1.Address,
Role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(3, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(1, 3),
},
},
},
},
{
role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(1, -1),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(1),
},
isWrite: false,
exp: []dax.AssignedNode{
{
Address: node1.Address,
Role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(1, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(1),
},
},
},
@ -1285,22 +1038,16 @@ func TestController(t *testing.T) {
{
// Ensure partitions are not returned sorted as strings.
role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(2, -1),
dax.NewVersionedPartition(10, -1),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(2, 10),
},
isWrite: false,
exp: []dax.AssignedNode{
{
Address: node0.Address,
Role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(2, 0),
dax.NewVersionedPartition(10, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(2, 10),
},
},
},

67
dax/mds/controller/naive/balancer.go
View file

@ -55,15 +55,18 @@ type WorkerJobService interface {
CreateJobs(ctx context.Context, balancerName string, worker dax.Worker, job ...dax.Job) error
DeleteJob(ctx context.Context, balancerName string, worker dax.Worker, job dax.Job) error
DeleteJobs(ctx context.Context, balancerName, prefix string) (InternalDiffs, error)
JobCounts(ctx context.Context, balancerName string, worker ...dax.Worker) (map[dax.Worker]int, error)
ListJobs(ctx context.Context, balancerName string, worker dax.Worker) (dax.Jobs, error)
}
// TODO: I don't think all these method names need "Free" in them.
type FreeJobService interface {
CreateFreeJobs(ctx context.Context, balancerName string, job ...dax.Job) error
DeleteFreeJob(ctx context.Context, balancerName string, job dax.Job) error
ListFreeJobs(ctx context.Context, balancerName string) (dax.Jobs, error)
MergeFreeJobs(ctx context.Context, balancerName string, jobs dax.Jobs) error
DeleteFreeJobs(ctx context.Context, balancerName, prefix string) error
}
// New returns a new instance of Balancer.
@ -90,15 +93,15 @@ func (b *Balancer) AddWorker(ctx context.Context, worker fmt.Stringer) ([]dax.Wo
return nil, errors.Wrap(err, "adding worker")
}
return diff.output(), nil
return diff.Output(), nil
}
func (b *Balancer) addWorker(ctx context.Context, worker dax.Worker) (internalDiffs, error) {
func (b *Balancer) addWorker(ctx context.Context, worker dax.Worker) (InternalDiffs, error) {
// If this worker already exists, don't do anything.
if exists, err := b.current.WorkerExists(ctx, b.name, worker); err != nil {
return nil, errors.Wrap(err, "checking if worker exists")
} else if exists {
return internalDiffs{}, nil
return InternalDiffs{}, nil
}
if err := b.current.CreateWorker(ctx, b.name, worker); err != nil {
@ -133,15 +136,15 @@ func (b *Balancer) RemoveWorker(ctx context.Context, worker fmt.Stringer) ([]dax
return nil, errors.Wrap(err, "removing worker")
}
return diff.output(), nil
return diff.Output(), nil
}
func (b *Balancer) removeWorker(ctx context.Context, worker dax.Worker) (internalDiffs, error) {
func (b *Balancer) removeWorker(ctx context.Context, worker dax.Worker) (InternalDiffs, error) {
// If this worker doesn't exist, don't do anything else.
if exists, err := b.current.WorkerExists(ctx, b.name, worker); err != nil {
return nil, errors.Wrap(err, "checking if worker exists")
} else if !exists {
return internalDiffs{}, nil
return InternalDiffs{}, nil
}
jobs, err := b.current.ListJobs(ctx, b.name, worker)
@ -162,9 +165,9 @@ func (b *Balancer) removeWorker(ctx context.Context, worker dax.Worker) (interna
// Even though this may not be useful to the caller (for example, in the
// case where the worker has died and no longer exists), return the diffs
// which represent the removal of jobs from the worker.
diff := newInternalDiffs()
diff := NewInternalDiffs()
for _, job := range jobs {
diff.removed(worker, job)
diff.Removed(worker, job)
}
return diff, nil
@ -198,10 +201,10 @@ func (b *Balancer) AddJobs(ctx context.Context, jobs ...fmt.Stringer) ([]dax.Wor
return nil, errors.Wrap(err, "adding job")
}
return diff.output(), nil
return diff.Output(), nil
}
func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (internalDiffs, error) {
func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (InternalDiffs, error) {
if cnt, err := b.current.WorkerCount(ctx, b.name); err != nil {
return nil, errors.Wrap(err, "getting worker count")
} else if cnt == 0 {
@ -210,7 +213,7 @@ func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (internalDiffs,
}
// TODO: we might want to inform the user that a job is in the free list
// because there are no workers.
return internalDiffs{}, nil
return InternalDiffs{}, nil
}
workerJobs, err := b.current.WorkersJobs(ctx, b.name)
@ -229,7 +232,7 @@ func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (internalDiffs,
jobCounts[v.ID] = len(v.Jobs)
}
diffs := newInternalDiffs()
diffs := NewInternalDiffs()
jobsToCreate := make(map[dax.Worker][]dax.Job)
@ -263,7 +266,7 @@ func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (internalDiffs,
return nil, errors.Wrap(err, "creating job")
}
for _, job := range jobs {
diffs.added(worker, job)
diffs.Added(worker, job)
}
}
@ -282,10 +285,24 @@ func (b *Balancer) RemoveJob(ctx context.Context, job fmt.Stringer) ([]dax.Worke
return nil, errors.Wrapf(err, "removing job: %s", job)
}
return diff.output(), nil
return diff.Output(), nil
}
func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (internalDiffs, error) {
func (b *Balancer) RemoveJobs(ctx context.Context, prefix string) ([]dax.WorkerDiff, error) {
b.mu.Lock()
defer b.mu.Unlock()
idiffs, err := b.current.DeleteJobs(ctx, b.name, prefix)
if err != nil {
return nil, errors.Wrap(err, "deleting worker jobs")
}
if err := b.freeJobs.DeleteFreeJobs(ctx, b.name, prefix); err != nil {
return nil, errors.Wrap(err, "deleting free jobs")
}
return idiffs.Output(), nil
}
func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (InternalDiffs, error) {
if worker, ok, err := b.workerForJob(ctx, job); err != nil {
return nil, errors.Wrapf(err, "getting worker for job: %s", job)
} else if ok {
@ -293,8 +310,8 @@ func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (internalDiffs, e
return nil, errors.Wrapf(err, "deleting job: %s", job)
}
diffs := newInternalDiffs()
diffs.removed(worker, job)
diffs := NewInternalDiffs()
diffs.Removed(worker, job)
return diffs, nil
}
@ -307,7 +324,7 @@ func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (internalDiffs, e
return nil, errors.Wrapf(err, "deleting free job: %s", job)
}
return internalDiffs{}, nil
return InternalDiffs{}, nil
}
// Balance ensures that all jobs are being handled by a worker by assigning jobs
@ -336,7 +353,7 @@ func (b *Balancer) Balance(ctx context.Context) ([]dax.WorkerDiff, error) {
return nil, errors.Wrap(err, "balancing jobs")
}
return diff.output(), nil
return diff.Output(), nil
}
// balance moves jobs among workers with the goal of having an equal number of
@ -346,7 +363,7 @@ func (b *Balancer) Balance(ctx context.Context) ([]dax.WorkerDiff, error) {
// the internalDiffs.merge() method, but we would need to modify that method to
// be smarter about the order in which it applies the add/remove operations.
// Until that's in place, we'll pass in a value here.
func (b *Balancer) balance(ctx context.Context, diffs internalDiffs) (internalDiffs, error) {
func (b *Balancer) balance(ctx context.Context, diffs InternalDiffs) (InternalDiffs, error) {
numWorkers, err := b.current.WorkerCount(ctx, b.name)
if err != nil {
return nil, errors.Wrapf(err, "getting worker count: %s", b.name)
@ -405,12 +422,12 @@ func (b *Balancer) balance(ctx context.Context, diffs internalDiffs) (internalDi
if rj, err := b.removeJob(ctx, sortedJobs[i]); err != nil {
return nil, errors.Wrapf(err, "removing job: %s", sortedJobs[i])
} else {
diffs.merge(rj)
diffs.Merge(rj)
}
if aj, err := b.addJobs(ctx, sortedJobs[i]); err != nil {
return nil, errors.Wrapf(err, "adding job: %s", sortedJobs[i])
} else {
diffs.merge(aj)
diffs.Merge(aj)
}
}
}
@ -547,8 +564,8 @@ func (b *Balancer) WorkersForJobPrefix(ctx context.Context, prefix string) ([]da
}
// processFreeJobs assigns all jobs in the free list to a worker.
func (b *Balancer) processFreeJobs(ctx context.Context) (internalDiffs, error) {
diffs := newInternalDiffs()
func (b *Balancer) processFreeJobs(ctx context.Context) (InternalDiffs, error) {
diffs := NewInternalDiffs()
jobs, err := b.freeJobs.ListFreeJobs(ctx, b.name)
if err != nil {
return nil, errors.Wrapf(err, "listing free jobs: %s", b.name)
@ -557,7 +574,7 @@ func (b *Balancer) processFreeJobs(ctx context.Context) (internalDiffs, error) {
if aj, err := b.addJobs(ctx, job); err != nil {
return nil, errors.Wrapf(err, "adding job: %s", job)
} else {
diffs.merge(aj)
diffs.Merge(aj)
}
if err := b.freeJobs.DeleteFreeJob(ctx, b.name, job); err != nil {
return nil, errors.Wrapf(err, "deleting free job: %s", job)

83
dax/mds/controller/naive/boltdb/balancer.go
View file

@ -289,6 +289,53 @@ func (w *workerJobService) DeleteJob(ctx context.Context, balancerName string, w
return tx.Commit()
}
func (w *workerJobService) DeleteJobs(ctx context.Context, balancerName, prefix string) (naive.InternalDiffs, error) {
tx, err := w.db.BeginTx(ctx, true)
if err != nil {
return nil, errors.Wrap(err, "beginning tx")
}
defer tx.Rollback()
bkt := tx.Bucket(bucketNaiveBalancer)
if bkt == nil {
return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer)
}
workers, err := w.getWorkers(ctx, tx, balancerName)
if err != nil {
return nil, errors.Wrap(err, "getting workers")
}
idiffs := naive.NewInternalDiffs()
for _, worker := range workers {
// get worker
wrkr := bkt.Get(workerKey(balancerName, worker))
if wrkr == nil {
panic("didn't find worker that should... definitely exist")
}
jobset, err := decodeJobSet(wrkr)
if err != nil {
return nil, errors.Wrap(err, "decoding job set")
}
jobs := jobset.RemovePrefix(prefix)
for _, job := range jobs {
idiffs.Removed(worker, job)
}
val, err := encodeJobSet(jobset)
if err != nil {
return nil, errors.Wrap(err, "encoding job set")
}
if err := bkt.Put(workerKey(balancerName, worker), val); err != nil {
return nil, errors.Wrap(err, "putting worker")
}
}
return idiffs, tx.Commit()
}
func (w *workerJobService) ListJobs(ctx context.Context, balancerName string, worker dax.Worker) (dax.Jobs, error) {
tx, err := w.db.BeginTx(ctx, false)
if err != nil {
@ -429,6 +476,42 @@ func (f *freeJobService) DeleteFreeJob(ctx context.Context, balancerName string,
return tx.Commit()
}
func (f *freeJobService) DeleteFreeJobs(ctx context.Context, balancerName, prefix string) error {
tx, err := f.db.BeginTx(ctx, true)
if err != nil {
return errors.Wrap(err, "beginning tx")
}
defer tx.Rollback()
bkt := tx.Bucket(bucketNaiveBalancer)
if bkt == nil {
return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer)
}
// get free jobs
fjs := bkt.Get(freeJobKey(balancerName))
if fjs == nil {
return nil
}
jobset, err := decodeJobSet(fjs)
if err != nil {
return errors.Wrap(err, "decoding job set")
}
jobset.RemovePrefix(prefix)
val, err := encodeJobSet(jobset)
if err != nil {
return errors.Wrap(err, "encoding job set")
}
if err := bkt.Put(freeJobKey(balancerName), val); err != nil {
return errors.Wrap(err, "putting free job")
}
return tx.Commit()
}
func (f *freeJobService) ListFreeJobs(ctx context.Context, balancerName string) (dax.Jobs, error) {
tx, err := f.db.BeginTx(ctx, false)
if err != nil {

16
dax/mds/controller/naive/types.go
View file

@ -20,13 +20,13 @@ func newJobSetDiffs() jobSetDiffs {
}
}
type internalDiffs map[dax.Worker]jobSetDiffs
type InternalDiffs map[dax.Worker]jobSetDiffs
func newInternalDiffs() internalDiffs {
return make(internalDiffs)
func NewInternalDiffs() InternalDiffs {
return make(InternalDiffs)
}
func (d internalDiffs) added(worker dax.Worker, job dax.Job) {
func (d InternalDiffs) Added(worker dax.Worker, job dax.Job) {
if _, ok := d[worker]; !ok {
d[worker] = newJobSetDiffs()
}
@ -39,7 +39,7 @@ func (d internalDiffs) added(worker dax.Worker, job dax.Job) {
d[worker].added.Add(job)
}
func (d internalDiffs) removed(worker dax.Worker, job dax.Job) {
func (d InternalDiffs) Removed(worker dax.Worker, job dax.Job) {
if _, ok := d[worker]; !ok {
d[worker] = newJobSetDiffs()
}
@ -52,7 +52,7 @@ func (d internalDiffs) removed(worker dax.Worker, job dax.Job) {
d[worker].removed.Add(job)
}
func (d internalDiffs) merge(d2 internalDiffs) {
func (d InternalDiffs) Merge(d2 InternalDiffs) {
for k, v := range d2 {
if _, ok := d[k]; !ok {
d[k] = newJobSetDiffs()
@ -62,9 +62,9 @@ func (d internalDiffs) merge(d2 internalDiffs) {
}
}
// output converts internalDiff to []controller.WorkerDiff for external
// Output converts internalDiff to []controller.WorkerDiff for external
// consumption.
func (d internalDiffs) output() []dax.WorkerDiff {
func (d InternalDiffs) Output() []dax.WorkerDiff {
out := make([]dax.WorkerDiff, len(d))
i := 0

28
dax/mds/controller/stringers.go
View file

@ -13,11 +13,11 @@ import (
// used as a job in the Balancer.
type pUnit struct {
t dax.TableKey
p dax.VersionedPartition
p dax.PartitionNum
}
func (p pUnit) String() string {
return fmt.Sprintf("%s|part_%d", p.t, p.p.Num)
return fmt.Sprintf("%s|part_%d", p.t, p.p)
}
func (p pUnit) table() dax.TableKey {
@ -25,14 +25,14 @@ func (p pUnit) table() dax.TableKey {
}
func (p pUnit) partitionNum() dax.PartitionNum {
return p.p.Num
return p.p
}
func partition(t dax.TableKey, p dax.VersionedPartition) pUnit {
func partition(t dax.TableKey, p dax.PartitionNum) pUnit {
return pUnit{t, p}
}
func partitions(t dax.TableKey, p ...dax.VersionedPartition) []pUnit {
func partitions(t dax.TableKey, p ...dax.PartitionNum) []pUnit {
ret := make([]pUnit, 0, len(p))
for _, vp := range p {
ret = append(ret, pUnit{t, vp})
@ -57,10 +57,7 @@ func decodePartition(j dax.Job) (pUnit, error) {
return pUnit{
t: dax.TableKey(parts[0]),
p: dax.VersionedPartition{
Num: dax.PartitionNum(intVar),
Version: -1,
},
p: dax.PartitionNum(intVar),
}, nil
}
@ -68,11 +65,11 @@ func decodePartition(j dax.Job) (pUnit, error) {
// a job in the Balancer.
type sUnit struct {
t dax.TableKey
s dax.VersionedShard
s dax.ShardNum
}
func (s sUnit) String() string {
return fmt.Sprintf("%s|shard_%s", s.t, s.s.Num)
return fmt.Sprintf("%s|shard_%s", s.t, s.s)
}
func (s sUnit) table() dax.TableKey {
@ -80,10 +77,10 @@ func (s sUnit) table() dax.TableKey {
}
func (s sUnit) shardNum() dax.ShardNum {
return s.s.Num
return s.s
}
func shard(t dax.TableKey, s dax.VersionedShard) sUnit {
func shard(t dax.TableKey, s dax.ShardNum) sUnit {
return sUnit{t, s}
}
@ -104,9 +101,6 @@ func decodeShard(j dax.Job) (sUnit, error) {
return sUnit{
t: dax.TableKey(parts[0]),
s: dax.VersionedShard{
Num: dax.ShardNum(uint64Var),
Version: -1,
},
s: dax.ShardNum(uint64Var),
}, nil
}

10
dax/role.go
View file

@ -33,8 +33,8 @@ var _ Role = &TranslateRole{}
// ComputeRole is a role specific to compute nodes.
type ComputeRole struct {
TableKey TableKey `json:"table-key"`
Shards VersionedShards `json:"shards"`
TableKey TableKey `json:"table-key"`
Shards ShardNums `json:"shards"`
}
// Type returns the type for ComputeRole. This is mainly to impolement the Role
@ -45,9 +45,9 @@ func (cr *ComputeRole) Type() RoleType {
// TranslateRole is a role specific to translate nodes.
type TranslateRole struct {
TableKey TableKey `json:"table-key"`
Partitions VersionedPartitions `json:"partitions"`
Fields VersionedFields `json:"fields"`
TableKey TableKey `json:"table-key"`
Partitions PartitionNums `json:"partitions"`
Fields []FieldName `json:"fields"`
}
// Type returns the type for TransteRole. This is mainly to impolement the Role

30
dax/versioned_field.go
View file

@ -1,30 +0,0 @@
package dax
import "fmt"
// VersionedField is used in a similar way to VersionedShard and
// VersionedPartition in that they all contain a snapshot version.
type VersionedField struct {
Name FieldName `json:"name"`
Version int `json:"version"`
}
// String returns the VersionedField (i.e. its Name and Version) as a string.
func (f VersionedField) String() string {
return fmt.Sprintf("%s.%d", f.Name, f.Version)
}
// NewVersionedField returns a VersionedField with the provided name and version.
func NewVersionedField(name FieldName, version int) VersionedField {
return VersionedField{
Name: name,
Version: version,
}
}
// VersionedFields is a sortable slice of VersionedField.
type VersionedFields []VersionedField
func (f VersionedFields) Len() int { return len(f) }
func (f VersionedFields) Less(i, j int) bool { return f[i].Name < f[j].Name }
func (f VersionedFields) Swap(i, j int) { f[i], f[j] = f[j], f[i] }

55
dax/versioned_partition.go
View file

@ -17,55 +17,10 @@ func (p PartitionNum) String() string {
return fmt.Sprintf("%d", p)
}
// VersionedPartition is a partition number along with the snapshot version
// which it is currently writing at.
type VersionedPartition struct {
Num PartitionNum `json:"num"`
Version int `json:"version"`
}
// NewVersionedPartition returns a VersionedPartition with the provided
// partition number and version.
func NewVersionedPartition(num PartitionNum, version int) VersionedPartition {
return VersionedPartition{
Num: num,
Version: version,
func NewPartitionNums(nums ...uint64) PartitionNums {
partitions := make(PartitionNums, len(nums))
for i, n := range nums {
partitions[i] = PartitionNum(n)
}
}
// String returns the VersionedPartition (i.e. its Num and Version) as a string.
func (p VersionedPartition) String() string {
return fmt.Sprintf("%d.%d", p.Num, p.Version)
}
// VersionedPartitions is a sortable slice of VersionedPartition.
type VersionedPartitions []VersionedPartition
func (p VersionedPartitions) Len() int { return len(p) }
func (p VersionedPartitions) Less(i, j int) bool { return p[i].Num < p[j].Num }
func (p VersionedPartitions) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
// NewVersionedPartitions returns the provided list of partition nums as a list
// of VersionedPartition with an invalid version (-1). This is to use for cases
// where the request should not be aware of a partition versioning.
func NewVersionedPartitions(partitionNums ...PartitionNum) VersionedPartitions {
pvs := make(VersionedPartitions, len(partitionNums))
for i := range partitionNums {
pvs[i] = VersionedPartition{
Num: partitionNums[i],
Version: -1,
}
}
return pvs
}
// Nums returns a slice of all the partition numbers in VersionedPartitions.
func (p VersionedPartitions) Nums() []PartitionNum {
pp := make([]PartitionNum, len(p))
for i := range p {
pp[i] = p[i].Num
}
return pp
return partitions
}

56
dax/versioned_shard.go
View file

@ -12,54 +12,14 @@ func (s ShardNum) String() string {
return fmt.Sprintf("%d", s)
}
// VersionedShard is a shard number along with the snapshot version which it is
// currently writing at.
type VersionedShard struct {
Num ShardNum `json:"num"`
Version int `json:"version"`
}
func (s ShardNums) Len() int { return len(s) }
func (s ShardNums) Less(i, j int) bool { return s[i] < s[j] }
func (s ShardNums) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// NewVersionedShard returns a VersionedShard with the provided shard number and version.
func NewVersionedShard(num ShardNum, version int) VersionedShard {
return VersionedShard{
Num: num,
Version: version,
func NewShardNums(nums ...uint64) ShardNums {
shards := make(ShardNums, len(nums))
for i, n := range nums {
shards[i] = ShardNum(n)
}
}
// String returns the VersionedShard (i.e. its Num and Version) as a string.
func (s VersionedShard) String() string {
return fmt.Sprintf("%d.%d", s.Num, s.Version)
}
// VersionedShards is a sortable slice of VersionedShard.
type VersionedShards []VersionedShard
func (s VersionedShards) Len() int { return len(s) }
func (s VersionedShards) Less(i, j int) bool { return s[i].Num < s[j].Num }
func (s VersionedShards) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// NewVersionedShards returns the provided list of shard nums as a list of
// VersionedShard with an invalid version (-1). This is to use for cases where
// the request should not be aware of shard versioning.
func NewVersionedShards(shardNums ...ShardNum) VersionedShards {
svs := make(VersionedShards, len(shardNums))
for i := range shardNums {
svs[i] = VersionedShard{
Num: shardNums[i],
Version: -1,
}
}
return svs
}
// Nums returns a slice of all the shard numbers in VersionedShards.
func (s VersionedShards) Nums() []ShardNum {
ss := make([]ShardNum, len(s))
for i := range s {
ss[i] = s[i].Num
}
return ss
return shards
}

110
dax/versionstore.go
View file

@ -1,110 +0,0 @@
package dax
import (
"context"
)
// VersionStore is an interface for tracking Shard, Partition, and Field[Key]
// versions. For example, when the contents of a shard are checkpointed, and a
// snapshot is generated, and the write log messages for that shard are
// truncated, the ShardVersion for that shard is incremented. The VersionStore
// is the interface through which various services read/write that version.
type VersionStore interface {
AddTable(ctx context.Context, qtid QualifiedTableID) error
RemoveTable(ctx context.Context, qtid QualifiedTableID) (VersionedShards, VersionedPartitions, error)
// Shards (shardData)
AddShards(ctx context.Context, qtid QualifiedTableID, shards ...VersionedShard) error
Shards(ctx context.Context, qtid QualifiedTableID) (VersionedShards, bool, error)
ShardVersion(ctx context.Context, qtid QualifiedTableID, shardNum ShardNum) (int, bool, error)
ShardTables(ctx context.Context, qual TableQualifier) (TableIDs, error)
// Partitions (tableKeys)
AddPartitions(ctx context.Context, qtid QualifiedTableID, partitions ...VersionedPartition) error
Partitions(ctx context.Context, qtid QualifiedTableID) (VersionedPartitions, bool, error)
PartitionVersion(ctx context.Context, qtid QualifiedTableID, partitionNum PartitionNum) (int, bool, error)
PartitionTables(ctx context.Context, qual TableQualifier) (TableIDs, error)
// Fields (fieldKeys)
AddFields(ctx context.Context, qtid QualifiedTableID, fields ...VersionedField) error
Fields(ctx context.Context, qtid QualifiedTableID) (VersionedFields, bool, error)
FieldVersion(ctx context.Context, qtid QualifiedTableID, field FieldName) (int, bool, error)
FieldTables(ctx context.Context, qual TableQualifier) (TableIDs, error)
Copy(ctx context.Context) (VersionStore, error)
}
type DirectiveVersion interface {
Increment(ctx context.Context, delta uint64) (uint64, error)
}
// Ensure type implements interface.
var _ VersionStore = (*nopVersionStore)(nil)
// nopVersionStore is a no-op implementation of the VersionStore interface.
type nopVersionStore struct{}
// NewNopVersionStore returns a new no-op instance of VersionStore.
func NewNopVersionStore() *nopVersionStore {
return &nopVersionStore{}
}
func (s *nopVersionStore) AddTable(ctx context.Context, qtid QualifiedTableID) error {
return nil
}
func (s *nopVersionStore) RemoveTable(ctx context.Context, qtid QualifiedTableID) (VersionedShards, VersionedPartitions, error) {
return nil, nil, nil
}
func (s *nopVersionStore) AddShards(ctx context.Context, qtid QualifiedTableID, shards ...VersionedShard) error {
return nil
}
func (s *nopVersionStore) Shards(ctx context.Context, qtid QualifiedTableID) (VersionedShards, bool, error) {
return nil, false, nil
}
func (s *nopVersionStore) ShardVersion(ctx context.Context, qtid QualifiedTableID, shardNum ShardNum) (int, bool, error) {
return 0, true, nil
}
func (s *nopVersionStore) ShardTables(ctx context.Context, qual TableQualifier) (TableIDs, error) {
return TableIDs{}, nil
}
func (s *nopVersionStore) AddPartitions(ctx context.Context, qtid QualifiedTableID, partitions ...VersionedPartition) error {
return nil
}
func (s *nopVersionStore) Partitions(ctx context.Context, qtid QualifiedTableID) (VersionedPartitions, bool, error) {
return nil, false, nil
}
func (s *nopVersionStore) PartitionVersion(ctx context.Context, qtid QualifiedTableID, partitionNum PartitionNum) (int, bool, error) {
return 0, true, nil
}
func (s *nopVersionStore) PartitionTables(ctx context.Context, qual TableQualifier) (TableIDs, error) {
return TableIDs{}, nil
}
func (s *nopVersionStore) AddFields(ctx context.Context, qtid QualifiedTableID, fields ...VersionedField) error {
return nil
}
func (s *nopVersionStore) Fields(ctx context.Context, qtid QualifiedTableID) (VersionedFields, bool, error) {
return nil, false, nil
}
func (s *nopVersionStore) FieldVersion(ctx context.Context, qtid QualifiedTableID, field FieldName) (int, bool, error) {
return 0, true, nil
}
func (s *nopVersionStore) FieldTables(ctx context.Context, qual TableQualifier) (TableIDs, error) {
return TableIDs{}, nil
}
func (s *nopVersionStore) Copy(ctx context.Context) (VersionStore, error) {
return nil, nil
}

20
dax/workerjob.go
View file

@ -2,8 +2,7 @@ package dax
import (
"sort"
"golang.org/x/exp/constraints"
"strings"
)
// Worker is a generic identifier used to represent a service responsible for
@ -78,10 +77,10 @@ func (w WorkerDiffs) Len() int { return len(w) }
func (w WorkerDiffs) Less(i, j int) bool { return w[i].WorkerID < w[j].WorkerID }
func (w WorkerDiffs) Swap(i, j int) { w[i], w[j] = w[j], w[i] }
// Set is a set of orderable items.
type Set[K constraints.Ordered] map[K]struct{}
// Set is a set of stringy items.
type Set[K ~string] map[K]struct{}
func NewSet[K constraints.Ordered](stuff ...K) Set[K] {
func NewSet[K ~string](stuff ...K) Set[K] {
s := make(map[K]struct{})
for _, thing := range stuff {
s[thing] = struct{}{}
@ -110,6 +109,17 @@ func (s Set[K]) Remove(k K) {
delete(s, k)
}
func (s Set[K]) RemovePrefix(prefix string) []K {
ret := make([]K, 0)
for k := range s {
if strings.HasPrefix(string(k), prefix) {
ret = append(ret, k)
delete(s, k)
}
}
return ret
}
// Slice returns a slice containing each member of the set in an undefined order.
func (s Set[K]) Slice() []K {
ret := make([]K, 0, len(s))

40
holder.go
View file

@ -147,8 +147,6 @@ type Holder struct {
// snapshotter/writelogger; then MDS should only start directing queries to
// that computer once it has completed applying the snapshot.
directiveApplied bool
versionStore dax.VersionStore
}
// HolderOpts holds information about the holder which other things might want
@ -344,8 +342,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
path: path,
indexes: make(map[string]*Index),
versionStore: dax.NewNopVersionStore(),
}
txf, err := NewTxFactory(cfg.StorageConfig.Backend, h.IndexesPath(), h)
@ -1018,14 +1014,6 @@ func (h *Holder) createIndex(cim *CreateIndexMessage, broadcast bool) (*Index, e
// Update options.
h.addIndex(index)
tkey := dax.TableKey(cim.Index)
qtid := tkey.QualifiedTableID()
// Initialize the table in holder.versionStore.
if err := h.versionStore.AddTable(context.Background(), qtid); err != nil {
h.Logger.Printf("could not add table to version store: %s", cim.Index)
}
if broadcast {
// Send the create index message to all nodes.
if err := h.broadcaster.SendSync(cim); err != nil {
@ -1045,7 +1033,7 @@ func (h *Holder) createIndex(cim *CreateIndexMessage, broadcast bool) (*Index, e
// createIndexWithPartitions is similar to createIndex, but it takes a list of
// partitions for which this node is responsible. This ensures that the node
// doesn't instantiate more partition TranslateStores than is necessary.
func (h *Holder) createIndexWithPartitions(cim *CreateIndexMessage, translatePartitions dax.VersionedPartitions) (*Index, error) {
func (h *Holder) createIndexWithPartitions(cim *CreateIndexMessage, translatePartitions dax.PartitionNums) (*Index, error) {
if cim.Index == "" {
return nil, errors.New("index name required")
}
@ -1068,24 +1056,6 @@ func (h *Holder) createIndexWithPartitions(cim *CreateIndexMessage, translatePar
// Update options.
h.addIndex(index)
tkey := dax.TableKey(cim.Index)
qtid := tkey.QualifiedTableID()
// Initialize the table in holder.versionStore.
if err := h.versionStore.AddTable(context.Background(), qtid); err != nil {
h.Logger.Printf("could not add table to version store: %s", cim.Index)
}
// Initialize a list of partitions at version 0.
newPartitions := make(dax.VersionedPartitions, len(translatePartitions))
for i := range translatePartitions {
newPartitions[i] = dax.NewVersionedPartition(translatePartitions[i].Num, 0)
}
if err := h.versionStore.AddPartitions(context.Background(), qtid, newPartitions...); err != nil {
return nil, errors.Wrap(err, "adding partitions to version store")
}
// Since this is a new index, we need to kick off
// its translation sync.
if err := h.translationSyncer.Reset(); err != nil {
@ -1242,14 +1212,6 @@ func (h *Holder) deleteIndex(name string) error {
// Remove reference.
h.deleteIndexFromMap(name)
tkey := dax.TableKey(name)
qtid := tkey.QualifiedTableID()
// Remove the index from holder.versionStore.
if _, _, err := h.versionStore.RemoveTable(context.Background(), qtid); err != nil {
h.Logger.Printf("could not find table to remove from version store: %s", name)
}
// I'm not sure if calling Reset() here is necessary
// since closing the index stops its translation
// sync processes.

4
index.go
View file

@ -50,7 +50,7 @@ type Index struct {
holder *Holder
// Per-partition translation stores
translatePartitions dax.VersionedPartitions
translatePartitions dax.PartitionNums
translateStores map[int]TranslateStore
translationSyncer TranslationSyncer
@ -977,7 +977,7 @@ func (i *Index) DeleteField(name string) error {
// case, we need to update this cached value. Really, this is kind of hacky and
// we need to revisit the ApplyDirective logic so that it's more intuitive with
// respect to index.translatePartitions.
func (i *Index) SetTranslatePartitions(tp dax.VersionedPartitions) {
func (i *Index) SetTranslatePartitions(tp dax.PartitionNums) {
i.mu.Lock()
defer i.mu.Unlock()

11
server.go
View file

@ -551,15 +551,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.holder.Logger.Infof("cwd: %v", cwd)
s.holder.Logger.Infof("cmd line: %v", strings.Join(os.Args, " "))
// The compute nodes keep a local cache of the VersionStore which applies
// only to the data (shard, partitions, fields) managed by the compute node
// (as opposed to the VersionStore in MDS which keeps information about all
// data). It would be okay for this to be an in-memory implementation as
// long as the compute node isn't expected to survive a restart; in that
// case, it would be necessary to use an implementation which saves state
// somewhere, such as local disk.
versionStore := inmem.NewVersionStore()
s.cluster.Path = path
s.cluster.logger = s.logger
s.cluster.holder = s.holder
@ -567,7 +558,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.cluster.noder = s.noder
s.cluster.sharder = s.sharder
s.cluster.serverlessStorage = s.serverlessStorage
s.cluster.versionStore = versionStore
// Append the NodeID tag to stats.
s.holder.Stats = s.holder.Stats.WithTags(fmt.Sprintf("node_id:%s", s.nodeID))
@ -583,7 +573,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.holder.broadcaster = s
s.holder.sharder = s.sharder
s.holder.serializer = s.serializer
s.holder.versionStore = versionStore
// Initial stats must be invoked after the executor obtains reference to the holder.
s.executor.InitStats()