diff --git a/api.go b/api.go index eb6f7f7be..608961872 100644 --- a/api.go +++ b/api.go @@ -716,29 +716,6 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string, } } -func (api *API) getOrCreateShardVersion(ctx context.Context, indexName string, shard uint64) (int, error) { - tableName := dax.TableName(indexName) - shardNum := dax.ShardNum(shard) - - // Here we assume that indexName is the string encoding of QualifiedTableID. - qtid, err := dax.QualifiedTableIDFromKey(indexName) - if err != nil { - return -1, errors.Wrap(err, "decoding qtid from key (indexName)") - } - - version, found, err := api.holder.versionStore.ShardVersion(ctx, qtid, shardNum) - if err != nil { - return -1, errors.Wrap(err, "getting shard version") - } else if !found { - version = 0 - api.server.logger.Printf("could not find version for shard: %s, %d, so creating 0", tableName, shardNum) - if err := api.holder.versionStore.AddShards(ctx, qtid, dax.NewVersionedShard(shardNum, version)); err != nil { - return -1, errors.Wrap(err, "adding shard 0") - } - } - return version, nil -} - // DeleteField removes the named field from the named index. If the index is not // found, an error is returned. If the field is not found, it is ignored and no // action is taken. @@ -3324,9 +3301,9 @@ var methodsNormal = map[apiMethod]struct{}{ apiDeleteDataframe: {}, } -func shardInShards(i dax.ShardNum, s dax.VersionedShards) bool { +func shardInShards(i dax.ShardNum, s dax.ShardNums) bool { for _, o := range s { - if i == o.Num { + if i == o { return true } } diff --git a/api_directive.go b/api_directive.go index 6476dec31..a5e8e72a2 100644 --- a/api_directive.go +++ b/api_directive.go @@ -345,7 +345,7 @@ func (api *API) pushJobsTableKeys(ctx context.Context, jobs chan<- directiveJobT jobs <- directiveJobTableKeys{ idx: idx, tkey: tkey, - partition: partition.Num, + partition: partition, } } } @@ -419,7 +419,7 @@ func (api *API) pushJobsFieldKeys(ctx context.Context, jobs chan<- directiveJobT for _, field := range fields { jobs <- directiveJobFieldKeys{ tkey: tkey, - field: field.Name, + field: field, } } } @@ -503,7 +503,7 @@ func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType for _, shard := range shards { jobs <- directiveJobShards{ tkey: tkey, - shard: shard.Num, + shard: shard, } } } @@ -710,11 +710,11 @@ func thingsAdded[K comparable](from []K, to []K) []K { // partitionsComparer is used to compare the differences between two maps of // table:[]partition. type partitionsComparer struct { - from map[dax.TableKey]dax.VersionedPartitions - to map[dax.TableKey]dax.VersionedPartitions + from map[dax.TableKey]dax.PartitionNums + to map[dax.TableKey]dax.PartitionNums } -func newPartitionsComparer(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.TableKey]dax.VersionedPartitions) *partitionsComparer { +func newPartitionsComparer(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) *partitionsComparer { return &partitionsComparer{ from: from, to: to, @@ -723,23 +723,23 @@ func newPartitionsComparer(from map[dax.TableKey]dax.VersionedPartitions, to map // added returns the partitions which are present in `to` but not in `from`. The // results remain in the format of a map of table:[]partition. -func (p *partitionsComparer) added() map[dax.TableKey]dax.VersionedPartitions { +func (p *partitionsComparer) added() map[dax.TableKey]dax.PartitionNums { return partitionsAdded(p.from, p.to) } // removed returns the partitions which are present in `from` but not in `to`. // The results remain in the format of a map of table:[]partition. -func (p *partitionsComparer) removed() map[dax.TableKey]dax.VersionedPartitions { +func (p *partitionsComparer) removed() map[dax.TableKey]dax.PartitionNums { return partitionsAdded(p.to, p.from) } // partitionsAdded returns the partitions which are present in `to` but not in `from`. -func partitionsAdded(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.TableKey]dax.VersionedPartitions) map[dax.TableKey]dax.VersionedPartitions { +func partitionsAdded(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) map[dax.TableKey]dax.PartitionNums { if from == nil { return to } - added := make(map[dax.TableKey]dax.VersionedPartitions) + added := make(map[dax.TableKey]dax.PartitionNums) for tt, tps := range to { fps, found := from[tt] if !found { @@ -747,7 +747,7 @@ func partitionsAdded(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.T continue } - addedPartitions := dax.VersionedPartitions{} + addedPartitions := dax.PartitionNums{} for i := range tps { var found bool for j := range fps { @@ -771,11 +771,11 @@ func partitionsAdded(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.T // fieldsComparer is used to compare the differences between two maps of // table:[]fieldVersion. type fieldsComparer struct { - from map[dax.TableKey]dax.VersionedFields - to map[dax.TableKey]dax.VersionedFields + from map[dax.TableKey][]dax.FieldName + to map[dax.TableKey][]dax.FieldName } -func newFieldsComparer(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]dax.VersionedFields) *fieldsComparer { +func newFieldsComparer(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) *fieldsComparer { return &fieldsComparer{ from: from, to: to, @@ -784,23 +784,23 @@ func newFieldsComparer(from map[dax.TableKey]dax.VersionedFields, to map[dax.Tab // added returns the fields which are present in `to` but not in `from`. The // results remain in the format of a map of table:[]field. -func (f *fieldsComparer) added() map[dax.TableKey]dax.VersionedFields { +func (f *fieldsComparer) added() map[dax.TableKey][]dax.FieldName { return fieldsAdded(f.from, f.to) } // removed returns the fields which are present in `from` but not in `to`. // The results remain in the format of a map of table:[]field. -func (f *fieldsComparer) removed() map[dax.TableKey]dax.VersionedFields { +func (f *fieldsComparer) removed() map[dax.TableKey][]dax.FieldName { return fieldsAdded(f.to, f.from) } // fieldsAdded returns the fields which are present in `to` but not in `from`. -func fieldsAdded(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]dax.VersionedFields) map[dax.TableKey]dax.VersionedFields { +func fieldsAdded(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) map[dax.TableKey][]dax.FieldName { if from == nil { return to } - added := make(map[dax.TableKey]dax.VersionedFields) + added := make(map[dax.TableKey][]dax.FieldName) for tt, tps := range to { fps, found := from[tt] if !found { @@ -808,7 +808,7 @@ func fieldsAdded(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey] continue } - addedFieldVersions := dax.VersionedFields{} + addedFieldVersions := []dax.FieldName{} for i := range tps { var found bool for j := range fps { @@ -832,11 +832,11 @@ func fieldsAdded(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey] // shardsComparer is used to compare the differences between two maps of // table:[]shardV. type shardsComparer struct { - from map[dax.TableKey]dax.VersionedShards - to map[dax.TableKey]dax.VersionedShards + from map[dax.TableKey]dax.ShardNums + to map[dax.TableKey]dax.ShardNums } -func newShardsComparer(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]dax.VersionedShards) *shardsComparer { +func newShardsComparer(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) *shardsComparer { return &shardsComparer{ from: from, to: to, @@ -845,23 +845,23 @@ func newShardsComparer(from map[dax.TableKey]dax.VersionedShards, to map[dax.Tab // added returns the shards which are present in `to` but not in `from`. The // results remain in the format of a map of table:[]shard. -func (s *shardsComparer) added() map[dax.TableKey]dax.VersionedShards { +func (s *shardsComparer) added() map[dax.TableKey]dax.ShardNums { return shardsAdded(s.from, s.to) } // removed returns the shards which are present in `from` but not in `to`. The // results remain in the format of a map of table:[]shard. -func (s *shardsComparer) removed() map[dax.TableKey]dax.VersionedShards { +func (s *shardsComparer) removed() map[dax.TableKey]dax.ShardNums { return shardsAdded(s.to, s.from) } // shardsAdded returns the shards which are present in `to` but not in `from`. -func shardsAdded(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]dax.VersionedShards) map[dax.TableKey]dax.VersionedShards { +func shardsAdded(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) map[dax.TableKey]dax.ShardNums { if from == nil { return to } - added := make(map[dax.TableKey]dax.VersionedShards) + added := make(map[dax.TableKey]dax.ShardNums) for tt, tss := range to { fss, found := from[tt] if !found { @@ -869,7 +869,7 @@ func shardsAdded(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey] continue } - addedShards := dax.VersionedShards{} + addedShards := dax.ShardNums{} for i := range tss { var found bool for j := range fss { @@ -892,7 +892,7 @@ func shardsAdded(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey] // createTableAndFields creates the FeatureBase Tables and Fields provided in // the dax.Directive format. -func (api *API) createTableAndFields(tbl *dax.QualifiedTable, partitions dax.VersionedPartitions) error { +func (api *API) createTableAndFields(tbl *dax.QualifiedTable, partitions dax.PartitionNums) error { cim := &CreateIndexMessage{ Index: string(tbl.Key()), CreatedAt: 0, diff --git a/cluster.go b/cluster.go index a49621338..b60174636 100644 --- a/cluster.go +++ b/cluster.go @@ -74,7 +74,6 @@ type cluster struct { // nolint: maligned partitionAssigner string serverlessStorage *storage.ResourceManager - versionStore dax.VersionStore // isComputeNode is set to true if this node is running as a DAX compute // node. @@ -1013,9 +1012,9 @@ type TransactionMessage struct { Action string } -func intInPartitions(i int, s dax.VersionedPartitions) bool { +func intInPartitions(i int, s dax.PartitionNums) bool { for _, a := range s { - if int(a.Num) == i { + if int(a) == i { return true } } diff --git a/dax/boltdb/versionstore.go b/dax/boltdb/versionstore.go deleted file mode 100644 index 91075823e..000000000 --- a/dax/boltdb/versionstore.go +++ /dev/null @@ -1,768 +0,0 @@ -package boltdb - -import ( - "bytes" - "context" - "encoding/binary" - "fmt" - "strconv" - "strings" - - "github.com/featurebasedb/featurebase/v3/dax" - "github.com/featurebasedb/featurebase/v3/dax/inmem" - "github.com/featurebasedb/featurebase/v3/errors" - "github.com/featurebasedb/featurebase/v3/logger" -) - -var ( - bucketTables = Bucket("versionStoreTables") - bucketShards = Bucket("versionStoreShards") - bucketTableKeys = Bucket("versionStoreTableKeys") - bucketFieldKeys = Bucket("versionStoreFieldKeys") -) - -// VersionStoreBuckets defines the buckets used by this package. It can be -// called during setup to create the buckets ahead of time. -var VersionStoreBuckets []Bucket = []Bucket{ - bucketTables, - bucketShards, - bucketTableKeys, - bucketFieldKeys, -} - -// Ensure type implements interface. -var _ dax.VersionStore = (*VersionStore)(nil) - -// VersionStore manages all version info for shard, table keys, and field keys. -type VersionStore struct { - db *DB - - logger logger.Logger -} - -// NewVersionStore returns a new instance of VersionStore with default values. -func NewVersionStore(db *DB, logger logger.Logger) *VersionStore { - return &VersionStore{ - db: db, - logger: logger, - } -} - -func (s *VersionStore) AddTable(ctx context.Context, qtid dax.QualifiedTableID) error { - tx, err := s.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "getting transaction") - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketTables) - if bkt == nil { - return errors.Errorf(ErrFmtBucketNotFound, bucketTables) - } - - if val := bkt.Get(tableKey(qtid)); val != nil { - return dax.NewErrTableIDExists(qtid) - } - - // The assumption is that we may store information about the table (other - // than just the fact that it exists). So for now, the value is an empty - // JSON object. - val := []byte("{}") - - if err := bkt.Put(tableKey(qtid), val); err != nil { - return errors.Wrap(err, "putting table") - } - - // Add the table to the "table index" of the other buckets. - // - // Shards - if bkt := tx.Bucket(bucketShards); bkt == nil { - return errors.Errorf(ErrFmtBucketNotFound, bucketShards) - } else if err := bkt.Put(tableKey(qtid), val); err != nil { - return errors.Wrap(err, "putting table into shards") - } - - // TableKeys. - if bkt := tx.Bucket(bucketTableKeys); bkt == nil { - return errors.Errorf(ErrFmtBucketNotFound, bucketTableKeys) - } else if err := bkt.Put(tableKey(qtid), val); err != nil { - return errors.Wrap(err, "putting table into table keys") - } - - // FieldKeys. - if bkt := tx.Bucket(bucketFieldKeys); bkt == nil { - return errors.Errorf(ErrFmtBucketNotFound, bucketFieldKeys) - } else if err := bkt.Put(tableKey(qtid), val); err != nil { - return errors.Wrap(err, "putting table into field keys") - } - - return tx.Commit() -} - -func (s *VersionStore) RemoveTable(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedShards, dax.VersionedPartitions, error) { - tx, err := s.db.BeginTx(ctx, true) - if err != nil { - return nil, nil, err - } - defer tx.Rollback() - - // Get the shards and partitions before deleting by table. - shards, err := s.getShards(ctx, tx, qtid) - if err != nil { - return nil, nil, err - } - - partitions, err := s.getPartitions(ctx, tx, qtid) - if err != nil { - return nil, nil, err - } - - if err := removeTable(ctx, tx, qtid); err != nil { - return nil, nil, err - } - - if err := tx.Commit(); err != nil { - return nil, nil, err - } - - return shards, partitions, nil -} - -func removeTable(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID) error { - // Tables. - if bkt := tx.Bucket(bucketTables); bkt == nil { - return errors.Errorf(ErrFmtBucketNotFound, bucketTables) - } else if err := bkt.Delete(tableKey(qtid)); err != nil { - return errors.Wrap(err, "deleting table") - } - - // Shards. - if bkt := tx.Bucket(bucketShards); bkt == nil { - return errors.Errorf(ErrFmtBucketNotFound, bucketShards) - } else if err := bkt.Delete(tableKey(qtid)); err != nil { - return errors.Wrap(err, "deleting table in shards") - } else if err := deleteByPrefix(tx, bucketShards, []byte(fmt.Sprintf(prefixFmtShards, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))); err != nil { - return errors.Wrap(err, "deleting shards for table") - } - - // TableKeys. - if bkt := tx.Bucket(bucketTableKeys); bkt == nil { - return errors.Errorf(ErrFmtBucketNotFound, bucketTableKeys) - } else if err := bkt.Delete(tableKey(qtid)); err != nil { - return errors.Wrap(err, "deleting table in table keys") - } else if err := deleteByPrefix(tx, bucketTableKeys, []byte(fmt.Sprintf(prefixFmtTableKeys, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))); err != nil { - return errors.Wrap(err, "deleting table keys for table") - } - - // FieldKeys. - if bkt := tx.Bucket(bucketFieldKeys); bkt == nil { - return errors.Errorf(ErrFmtBucketNotFound, bucketFieldKeys) - } else if err := bkt.Delete(tableKey(qtid)); err != nil { - return errors.Wrap(err, "deleting table in field keys") - } else if err := deleteByPrefix(tx, bucketFieldKeys, []byte(fmt.Sprintf(prefixFmtFieldKeys, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))); err != nil { - return errors.Wrap(err, "deleting field keys for table") - } - - return nil -} - -func deleteByPrefix(tx *Tx, bucket Bucket, prefix []byte) error { - bkt := tx.Bucket(bucket) - cursor := bkt.Cursor() - - // Deleting keys within the for loop seems to cause Next() to skip the next - // matching key because the Delete() call pops the item and effectively - // moves the cursor forward. Then calling Next() skips the item that was - // being pointed to after the delete. So, we're going to make a list of keys - // to delete, and then delete them outside of the cursor logic. - var keysToDelete [][]byte - for k, _ := cursor.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = cursor.Next() { - keysToDelete = append(keysToDelete, k) - } - - for _, k := range keysToDelete { - if err := bkt.Delete(k); err != nil { - return errors.Wrapf(err, "deleting key: %s", k) - } - } - - return nil -} - -func (s *VersionStore) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.VersionedShard) error { - tx, err := s.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "getting transaction") - } - defer tx.Rollback() - - for _, shard := range shards { - if err := createShard(ctx, tx, qtid, shard); err != nil { - return errors.Wrap(err, "creating shard") - } - } - - return tx.Commit() -} - -func createShard(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, shard dax.VersionedShard) error { - // TODO: validate data more formally - if shard.Version < 0 { - return errors.New(errors.ErrUncoded, fmt.Sprintf("invalid shard version: %d", shard.Version)) - } - - bkt := tx.Bucket(bucketShards) - if bkt == nil { - return errors.Errorf(ErrFmtBucketNotFound, bucketShards) - } - - // Ensure the table exists. - if val := bkt.Get(tableKey(qtid)); val == nil { - return dax.NewErrTableIDDoesNotExist(qtid) - } - - vsn := make([]byte, 8) - binary.LittleEndian.PutUint64(vsn, uint64(shard.Version)) - - return bkt.Put(shardKey(qtid, shard.Num), vsn) -} - -func (s *VersionStore) Shards(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedShards, bool, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return nil, false, errors.Wrap(err, "getting tx") - } - defer tx.Rollback() - - shards, err := s.getShards(ctx, tx, qtid) - if err != nil { - return nil, false, errors.Wrap(err, "getting shards") - } - - return shards, true, nil -} - -func (s *VersionStore) getShards(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID) (dax.VersionedShards, error) { - c := tx.Bucket(bucketShards).Cursor() - - // Deserialize rows into Shard objects. - shards := make(dax.VersionedShards, 0) - - prefix := []byte(fmt.Sprintf(prefixFmtShards, qtid.OrganizationID, qtid.DatabaseID, qtid.ID)) - for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { - if v == nil { - s.logger.Printf("nil value for key: %s", k) - continue - } - - var shard dax.VersionedShard - - shardNum, err := keyShardNum(k) - if err != nil { - return nil, errors.Wrapf(err, "getting shardNum from key: %v", k) - } - - shard.Num = shardNum - shard.Version = int(binary.LittleEndian.Uint64(v)) - - shards = append(shards, shard) - } - - return shards, nil -} - -// ShardVersion return the current version for the given table/shardNum. -// If a version is not being tracked, it returns a bool value of false. -func (s *VersionStore) ShardVersion(ctx context.Context, qtid dax.QualifiedTableID, shardNum dax.ShardNum) (int, bool, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return -1, false, err - } - defer tx.Rollback() - - return getShardVersion(ctx, tx, qtid, shardNum) -} - -func getShardVersion(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, shardNum dax.ShardNum) (int, bool, error) { - version := -1 - - bkt := tx.Bucket(bucketShards) - if bkt == nil { - return version, false, errors.Errorf(ErrFmtBucketNotFound, bucketShards) - } - - b := bkt.Get(shardKey(qtid, shardNum)) - if b == nil { - return version, false, nil - } - version = int(binary.LittleEndian.Uint64(b)) - - return version, true, nil -} - -func (s *VersionStore) ShardTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - return s.getTableIDs(ctx, tx, qual, bucketShards) -} - -func (s *VersionStore) getTableIDs(ctx context.Context, tx *Tx, qual dax.TableQualifier, bucket Bucket) (dax.TableIDs, error) { - c := tx.Bucket(bucket).Cursor() - - // Deserialize rows into Tables objects. - tableIDs := make(dax.TableIDs, 0) - - prefix := []byte(fmt.Sprintf(prefixFmtTables, qual.OrganizationID, qual.DatabaseID)) - for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { - if v == nil { - s.logger.Printf("nil value for key: %s", k) - continue - } - - var tableID dax.TableID - - tableID, err := keyTableID(k) - if err != nil { - return nil, errors.Wrapf(err, "getting table name from key: %v", k) - } - - tableIDs = append(tableIDs, tableID) - } - - return tableIDs, nil -} - -func (s *VersionStore) bucketTables(ctx context.Context, bucket Bucket) ([]dax.QualifiedTableID, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - c := tx.Bucket(bucket).Cursor() - - // Deserialize rows into Tables objects. - qtids := make([]dax.QualifiedTableID, 0) - - prefix := []byte(prefixTables) - for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { - if v == nil { - s.logger.Printf("nil value for key: %s", k) - continue - } - - qtid, err := keyQualifiedTableID(k) - if err != nil { - return nil, errors.Wrapf(err, "getting qualified table id from key: %v", k) - } - - qtids = append(qtids, qtid) - } - - return qtids, nil -} - -// AddPartitions adds new partitions to be managed by VersionStore. It returns -// the number of partitions added or an error. -func (s *VersionStore) AddPartitions(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.VersionedPartition) error { - tx, err := s.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "getting transaction") - } - defer tx.Rollback() - - for _, partition := range partitions { - if err := createPartition(ctx, tx, qtid, partition); err != nil { - return errors.Wrap(err, "creating partition") - } - } - - return tx.Commit() -} - -func createPartition(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, partition dax.VersionedPartition) error { - // TODO: validate data more formally - if partition.Version < 0 { - return errors.New(errors.ErrUncoded, fmt.Sprintf("invalid partition version: %d", partition.Version)) - } - - bkt := tx.Bucket(bucketTableKeys) - if bkt == nil { - return errors.Errorf(ErrFmtBucketNotFound, bucketTableKeys) - } - - // Ensure the table exists. - if val := bkt.Get(tableKey(qtid)); val == nil { - return dax.NewErrTableIDDoesNotExist(qtid) - } - - vsn := make([]byte, 8) - binary.LittleEndian.PutUint64(vsn, uint64(partition.Version)) - - return bkt.Put(partitionKey(qtid, partition.Num), vsn) -} - -func (s *VersionStore) Partitions(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedPartitions, bool, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return nil, false, errors.Wrap(err, "getting tx") - } - defer tx.Rollback() - - partitions, err := s.getPartitions(ctx, tx, qtid) - if err != nil { - return nil, false, errors.Wrap(err, "getting partitions") - } - - return partitions, true, nil -} - -func (s *VersionStore) getPartitions(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID) (dax.VersionedPartitions, error) { - c := tx.Bucket(bucketTableKeys).Cursor() - - // Deserialize rows into Partition objects. - partitions := make(dax.VersionedPartitions, 0) - - prefix := []byte(fmt.Sprintf(prefixFmtTableKeys, qtid.OrganizationID, qtid.DatabaseID, qtid.ID)) - for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { - if v == nil { - s.logger.Printf("nil value for key: %s", k) - continue - } - - var partition dax.VersionedPartition - - partitionNum, err := keyPartitionNum(k) - if err != nil { - return nil, errors.Wrapf(err, "getting partitionNum from key: %v", k) - } - - partition.Num = partitionNum - partition.Version = int(binary.LittleEndian.Uint64(v)) - - partitions = append(partitions, partition) - } - - return partitions, nil -} - -func (s *VersionStore) PartitionVersion(ctx context.Context, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) (int, bool, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return -1, false, err - } - defer tx.Rollback() - - return getPartitionVersion(ctx, tx, qtid, partitionNum) -} - -func getPartitionVersion(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) (int, bool, error) { - version := -1 - - bkt := tx.Bucket(bucketTableKeys) - if bkt == nil { - return version, false, errors.Errorf(ErrFmtBucketNotFound, bucketTableKeys) - } - - b := bkt.Get(partitionKey(qtid, partitionNum)) - if b == nil { - return version, false, nil - } - version = int(binary.LittleEndian.Uint64(b)) - - return version, true, nil -} - -func (s *VersionStore) PartitionTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - return s.getTableIDs(ctx, tx, qual, bucketTableKeys) -} - -// AddFields adds new fields to be managed by VersionStore. It returns the -// number of fields added or an error. -func (s *VersionStore) AddFields(ctx context.Context, qtid dax.QualifiedTableID, fields ...dax.VersionedField) error { - tx, err := s.db.BeginTx(ctx, true) - if err != nil { - return err - } - defer tx.Rollback() - - for _, field := range fields { - if err := createFieldVersion(ctx, tx, qtid, field); err != nil { - return errors.Wrap(err, "creating field version") - } - } - - return tx.Commit() -} - -func createFieldVersion(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, field dax.VersionedField) error { - // TODO: validate data more formally - if field.Version < 0 { - return errors.New(errors.ErrUncoded, fmt.Sprintf("invalid field version: %d", field.Version)) - } - - bkt := tx.Bucket(bucketFieldKeys) - if bkt == nil { - return errors.Errorf(ErrFmtBucketNotFound, bucketFieldKeys) - } - - // Ensure the table exists. - if val := bkt.Get(tableKey(qtid)); val == nil { - return dax.NewErrTableIDDoesNotExist(qtid) - } - - vsn := make([]byte, 8) - binary.LittleEndian.PutUint64(vsn, uint64(field.Version)) - - return bkt.Put(fieldKey(qtid, field.Name), vsn) -} - -func (s *VersionStore) Fields(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedFields, bool, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return nil, false, errors.Wrap(err, "getting tx") - } - defer tx.Rollback() - - fields, err := s.getFields(ctx, tx, qtid) - if err != nil { - return nil, false, errors.Wrap(err, "getting fields") - } - - return fields, true, nil -} - -func (s *VersionStore) getFields(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID) (dax.VersionedFields, error) { - c := tx.Bucket(bucketFieldKeys).Cursor() - - // Deserialize rows into FieldVersion objects. - fieldVersions := make(dax.VersionedFields, 0) - - prefix := []byte(fmt.Sprintf(prefixFmtFieldKeys, qtid.OrganizationID, qtid.DatabaseID, qtid.ID)) - for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { - if v == nil { - s.logger.Printf("nil value for key: %s", k) - continue - } - - var fieldVersion dax.VersionedField - - fieldName, err := keyFieldName(k) - if err != nil { - return nil, errors.Wrapf(err, "getting partitionNum from key: %v", k) - } - - fieldVersion.Name = fieldName - fieldVersion.Version = int(binary.LittleEndian.Uint64(v)) - - fieldVersions = append(fieldVersions, fieldVersion) - } - - return fieldVersions, nil -} - -func (s *VersionStore) FieldVersion(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName) (int, bool, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return -1, false, err - } - defer tx.Rollback() - - return getFieldVersion(ctx, tx, qtid, field) -} - -func getFieldVersion(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, field dax.FieldName) (int, bool, error) { - version := -1 - - bkt := tx.Bucket(bucketFieldKeys) - if bkt == nil { - return version, false, errors.Errorf(ErrFmtBucketNotFound, bucketFieldKeys) - } - - b := bkt.Get(fieldKey(qtid, field)) - if b == nil { - return version, false, nil - } - version = int(binary.LittleEndian.Uint64(b)) - - return version, true, nil -} - -func (s *VersionStore) FieldTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - return s.getTableIDs(ctx, tx, qual, bucketFieldKeys) -} - -// Copy returns an in-memory copy of VersionStore. -func (s *VersionStore) Copy(ctx context.Context) (dax.VersionStore, error) { - new := inmem.NewVersionStore() - - // shards. - qtids, err := s.bucketTables(ctx, bucketShards) - if err != nil { - return nil, errors.Wrap(err, "getting shard tables") - } - for _, qtid := range qtids { - shards, found, err := s.Shards(ctx, qtid) - if err != nil { - return nil, errors.Wrap(err, "getting shards") - } else if !found { - continue - } - _ = new.AddTable(ctx, qtid) - new.AddShards(ctx, qtid, shards...) - } - - // tableKeys. - qtids, err = s.bucketTables(ctx, bucketTableKeys) - if err != nil { - return nil, errors.Wrap(err, "getting table key tables") - } - for _, qtid := range qtids { - partitions, found, err := s.Partitions(ctx, qtid) - if err != nil { - return nil, errors.Wrap(err, "getting partitions") - } else if !found { - continue - } - _ = new.AddTable(ctx, qtid) - new.AddPartitions(ctx, qtid, partitions...) - } - - // fieldKeys. - qtids, err = s.bucketTables(ctx, bucketFieldKeys) - if err != nil { - return nil, errors.Wrap(err, "getting field key tables") - } - for _, qtid := range qtids { - fields, found, err := s.Fields(ctx, qtid) - if err != nil { - return nil, errors.Wrap(err, "getting fields") - } else if !found { - continue - } - _ = new.AddTable(ctx, qtid) - new.AddFields(ctx, qtid, fields...) - } - - return new, nil -} - -///////////////////////////////////////////////////////// - -const ( - prefixShards = "shards/" - prefixFmtShards = prefixShards + "%s/%s/%s/" - - prefixTableKeys = "tablekeys/" - prefixFmtTableKeys = prefixTableKeys + "%s/%s/%s/" - - prefixFieldKeys = "fieldkeys/" - prefixFmtFieldKeys = prefixFieldKeys + "%s/%s/%s/" - - prefixTables = "tables/" - prefixFmtTables = prefixTables + "%s/%s/" -) - -// tableKey returns a key based on table name. -func tableKey(qtid dax.QualifiedTableID) []byte { - qual := qtid.TableQualifier - key := fmt.Sprintf(prefixFmtTables+"%s", qual.OrganizationID, qual.DatabaseID, qtid.ID) - return []byte(key) -} - -// keyTableID gets the table ID out of the key. -func keyTableID(key []byte) (dax.TableID, error) { - parts := strings.Split(string(key), "/") - if len(parts) != 4 { - return "", errors.New(errors.ErrUncoded, "table key format expected: `tables/orgID/dbID/tableID`") - } - - return dax.TableID(parts[3]), nil -} - -// keyQualifiedTableID gets the qualified table ID out of the key. -func keyQualifiedTableID(key []byte) (dax.QualifiedTableID, error) { - parts := strings.Split(string(key), "/") - if len(parts) != 4 { - return dax.QualifiedTableID{}, errors.New(errors.ErrUncoded, "table key format expected: `tables/orgID/dbID/tableID`") - } - - return dax.NewQualifiedTableID( - dax.NewTableQualifier(dax.OrganizationID(parts[1]), dax.DatabaseID(parts[2])), - dax.TableID(parts[3]), - ), nil -} - -// shardKey returns a key based on table and shard. -func shardKey(qtid dax.QualifiedTableID, shard dax.ShardNum) []byte { - key := fmt.Sprintf(prefixFmtShards+"%d", qtid.OrganizationID, qtid.DatabaseID, qtid.ID, shard) - return []byte(key) -} - -// keyShardNum gets the shardNum out of the key. -func keyShardNum(key []byte) (dax.ShardNum, error) { - parts := strings.Split(string(key), "/") - if len(parts) != 5 { - return 0, errors.New(errors.ErrUncoded, "shard key format expected: `shards/orgID/dbID/table/shard`") - } - - intVar, err := strconv.Atoi(parts[4]) - if err != nil { - return 0, errors.Wrapf(err, "converting string to shardNum: %s", parts[4]) - } - - return dax.ShardNum(intVar), nil -} - -// partitionKey returns a key based on table and partition. -func partitionKey(qtid dax.QualifiedTableID, partition dax.PartitionNum) []byte { - key := fmt.Sprintf(prefixFmtTableKeys+"%d", qtid.OrganizationID, qtid.DatabaseID, qtid.ID, partition) - return []byte(key) -} - -// keyPartitionNum gets the partitionNum out of the key. -func keyPartitionNum(key []byte) (dax.PartitionNum, error) { - parts := strings.Split(string(key), "/") - if len(parts) != 5 { - return 0, errors.New(errors.ErrUncoded, "partition key format expected: `tablekeys/orgID/dbID/table/partition`") - } - - intVar, err := strconv.Atoi(parts[4]) - if err != nil { - return 0, errors.Wrapf(err, "converting string to partitionNum: %s", parts[4]) - } - - return dax.PartitionNum(intVar), nil -} - -// fieldKey returns a key based on table and field. -func fieldKey(qtid dax.QualifiedTableID, field dax.FieldName) []byte { - key := fmt.Sprintf(prefixFmtFieldKeys+"%s", qtid.OrganizationID, qtid.DatabaseID, qtid.ID, field) - return []byte(key) -} - -// keyFieldName gets the fieldName out of the key. -func keyFieldName(key []byte) (dax.FieldName, error) { - parts := strings.Split(string(key), "/") - if len(parts) != 5 { - return "", errors.New(errors.ErrUncoded, "field key format expected: `fieldkeys/orgID/dbID/table/field`") - } - - return dax.FieldName(parts[4]), nil -} diff --git a/dax/boltdb/versionstore_test.go b/dax/boltdb/versionstore_test.go deleted file mode 100644 index 6ad687a69..000000000 --- a/dax/boltdb/versionstore_test.go +++ /dev/null @@ -1,388 +0,0 @@ -package boltdb_test - -import ( - "context" - "fmt" - "sort" - "testing" - - "github.com/featurebasedb/featurebase/v3/dax" - "github.com/featurebasedb/featurebase/v3/dax/boltdb" - testbolt "github.com/featurebasedb/featurebase/v3/dax/test/boltdb" - "github.com/featurebasedb/featurebase/v3/logger" - "github.com/stretchr/testify/assert" -) - -func TestVersionStore(t *testing.T) { - db := testbolt.MustOpenDB(t) - defer testbolt.MustCloseDB(t, db) - - ctx := context.Background() - - t.Cleanup(func() { - testbolt.CleanupDB(t, db.Path()) - }) - - orgID := dax.OrganizationID("acme") - dbID := dax.DatabaseID("db1") - - qual := dax.NewTableQualifier(orgID, dbID) - - // Initialize the buckets. - assert.NoError(t, db.InitializeBuckets(boltdb.VersionStoreBuckets...)) - - t.Run("Tables", func(t *testing.T) { - vs := boltdb.NewVersionStore(db, logger.NopLogger) - - qtids := newQualifiedTableIDs(t, qual, 3) - qtid1 := qtids[0] - qtid2 := qtids[1] - qtid3 := qtids[2] - defer vs.RemoveTable(ctx, qtid1) - defer vs.RemoveTable(ctx, qtid2) - defer vs.RemoveTable(ctx, qtid3) - - // Add table 1. - assert.NoError(t, vs.AddTable(ctx, qtid1)) - - // Add table 2. - assert.NoError(t, vs.AddTable(ctx, qtid2)) - - // Add table 3. - assert.NoError(t, vs.AddTable(ctx, qtid3)) - }) - - t.Run("Shards", func(t *testing.T) { - vs := boltdb.NewVersionStore(db, logger.NopLogger) - - qtids := newQualifiedTableIDs(t, qual, 3) - qtid1 := qtids[0] - qtid2 := qtids[1] - qtid3 := qtids[2] - - // Add tables. - assert.NoError(t, vs.AddTable(ctx, qtid1)) - assert.NoError(t, vs.AddTable(ctx, qtid2)) - assert.NoError(t, vs.AddTable(ctx, qtid3)) - defer vs.RemoveTable(ctx, qtid1) - defer vs.RemoveTable(ctx, qtid2) - defer vs.RemoveTable(ctx, qtid3) - - // Create some shards to insert into the table. - shards := make(dax.VersionedShards, 3) - for i := range shards { - shards[i] = dax.VersionedShard{ - Num: dax.ShardNum(i), - Version: i * 2, - } - } - - // Add shards to table 1. - { - err := vs.AddShards(ctx, qtid1, shards...) - assert.NoError(t, err) - } - - // Add shards to table 2. - { - err := vs.AddShards(ctx, qtid2, shards...) - assert.NoError(t, err) - } - - // Fetch a shard and compare. - { - version, found, err := vs.ShardVersion(ctx, qtid1, 2) - assert.NoError(t, err) - assert.True(t, found) - assert.Equal(t, 4, version) - } - - // Fetch all shards and compare. - { - shrds, found, err := vs.Shards(ctx, qtid1) - assert.NoError(t, err) - assert.True(t, found) - assert.Equal(t, shards, shrds) - } - - // Fetch tables. - { - tblIDs, err := vs.ShardTables(ctx, qual) - assert.NoError(t, err) - exp := dax.TableIDs{qtid1.ID, qtid2.ID, qtid3.ID} - assert.Equal(t, exp, tblIDs) - } - - // Remove table 1. - { - shards, partitions, err := vs.RemoveTable(ctx, qtid1) - assert.NoError(t, err) - assert.Equal(t, shards, shards) - assert.Equal(t, dax.VersionedPartitions{}, partitions) - } - - // Fetch all shards and compare. - { - shrds, found, err := vs.Shards(ctx, qtid1) - assert.NoError(t, err) - assert.True(t, found) - assert.Equal(t, dax.VersionedShards{}, shrds) - } - }) - - t.Run("Partitions", func(t *testing.T) { - vs := boltdb.NewVersionStore(db, logger.NopLogger) - - // Create some partitions to insert into the table. - partitions := make(dax.VersionedPartitions, 3) - for i := range partitions { - partitions[i] = dax.VersionedPartition{ - Num: dax.PartitionNum(i), - Version: i * 2, - } - } - - qtids := newQualifiedTableIDs(t, qual, 2) - qtid1 := qtids[0] - qtid2 := qtids[1] - - // Add tables. - assert.NoError(t, vs.AddTable(ctx, qtid1)) - assert.NoError(t, vs.AddTable(ctx, qtid2)) - defer vs.RemoveTable(ctx, qtid1) - defer vs.RemoveTable(ctx, qtid2) - - // Add partitions to table 1. - { - err := vs.AddPartitions(ctx, qtid1, partitions...) - assert.NoError(t, err) - } - - // Add partitions to table 2. - { - err := vs.AddPartitions(ctx, qtid2, partitions...) - assert.NoError(t, err) - } - - // Fetch a partition and compare. - { - version, found, err := vs.PartitionVersion(ctx, qtid1, 2) - assert.NoError(t, err) - assert.True(t, found) - assert.Equal(t, 4, version) - } - - // Fetch all partitions and compare. - { - parts, found, err := vs.Partitions(ctx, qtid1) - assert.NoError(t, err) - assert.True(t, found) - assert.Equal(t, partitions, parts) - } - - // Fetch tables. - { - tblIDs, err := vs.PartitionTables(ctx, qual) - assert.NoError(t, err) - exp := dax.TableIDs{qtid1.ID, qtid2.ID} - assert.Equal(t, exp, tblIDs) - } - - // Remove table 1. - { - shards, partitions, err := vs.RemoveTable(ctx, qtid1) - assert.NoError(t, err) - assert.Equal(t, dax.VersionedShards{}, shards) - assert.Equal(t, partitions, partitions) - } - - // Fetch all partitions and compare. - { - parts, found, err := vs.Partitions(ctx, qtid1) - assert.NoError(t, err) - assert.True(t, found) - assert.Equal(t, dax.VersionedPartitions{}, parts) - } - }) - - t.Run("FieldVersions", func(t *testing.T) { - vs := boltdb.NewVersionStore(db, logger.NopLogger) - - qtids := newQualifiedTableIDs(t, qual, 2) - qtid1 := qtids[0] - qtid2 := qtids[1] - - // Add tables. - assert.NoError(t, vs.AddTable(ctx, qtid1)) - assert.NoError(t, vs.AddTable(ctx, qtid2)) - defer vs.RemoveTable(ctx, qtid1) - defer vs.RemoveTable(ctx, qtid2) - - // Create some fieldVersions to insert into the table. - fieldVersions := make(dax.VersionedFields, 3) - for i := range fieldVersions { - fieldVersions[i] = dax.VersionedField{ - Name: dax.FieldName(fmt.Sprintf("fld-%d", i)), - Version: i * 2, - } - } - - // Add fieldVersions to table 1. - { - err := vs.AddFields(ctx, qtid1, fieldVersions...) - assert.NoError(t, err) - } - - // Add fieldVersions to table 2. - { - err := vs.AddFields(ctx, qtid2, fieldVersions...) - assert.NoError(t, err) - } - - // Fetch a fieldVersion and compare. - { - version, found, err := vs.FieldVersion(ctx, qtid1, dax.FieldName("fld-2")) - assert.NoError(t, err) - assert.True(t, found) - assert.Equal(t, 4, version) - } - - // Fetch all fieldVersions and compare. - { - flds, found, err := vs.Fields(ctx, qtid1) - assert.NoError(t, err) - assert.True(t, found) - assert.Equal(t, fieldVersions, flds) - } - - // Fetch tables. - { - tblIDs, err := vs.FieldTables(ctx, qual) - assert.NoError(t, err) - exp := dax.TableIDs{qtid1.ID, qtid2.ID} - assert.Equal(t, exp, tblIDs) - } - - // Remove table 1. - { - shards, partitions, err := vs.RemoveTable(ctx, qtid1) - assert.NoError(t, err) - assert.Equal(t, dax.VersionedShards{}, shards) - assert.Equal(t, dax.VersionedPartitions{}, partitions) - } - - // Fetch all fieldVersions and compare. - { - flds, found, err := vs.Fields(ctx, qtid1) - assert.NoError(t, err) - assert.True(t, found) - assert.Equal(t, dax.VersionedFields{}, flds) - } - }) - - t.Run("Copy", func(t *testing.T) { - vs := boltdb.NewVersionStore(db, logger.NopLogger) - - qtids := newQualifiedTableIDs(t, qual, 1) - qtid1 := qtids[0] - - // Add tables. - assert.NoError(t, vs.AddTable(ctx, qtid1)) - defer vs.RemoveTable(ctx, qtid1) - - // Create some shards to insert into the table. - shards := make(dax.VersionedShards, 3) - for i := range shards { - shards[i] = dax.VersionedShard{ - Num: dax.ShardNum(i), - Version: i * 2, - } - } - - // Create some partitions to insert into the table. - partitions := make(dax.VersionedPartitions, 3) - for i := range partitions { - partitions[i] = dax.VersionedPartition{ - Num: dax.PartitionNum(i), - Version: i * 2, - } - } - - // Create some fieldVersions to insert into the table. - fieldVersions := make(dax.VersionedFields, 3) - for i := range fieldVersions { - fieldVersions[i] = dax.VersionedField{ - Name: dax.FieldName(fmt.Sprintf("fld-%d", i)), - Version: i * 2, - } - } - - // Add shards to table 1. - { - err := vs.AddShards(ctx, qtid1, shards...) - assert.NoError(t, err) - } - - // Add partitions to table 1. - { - err := vs.AddPartitions(ctx, qtid1, partitions...) - assert.NoError(t, err) - } - - // Add fieldVersions to table 1. - { - err := vs.AddFields(ctx, qtid1, fieldVersions...) - assert.NoError(t, err) - } - - copy, err := vs.Copy(ctx) - assert.NoError(t, err) - - // Fetch a shard and compare. - { - version, found, err := copy.ShardVersion(ctx, qtid1, 2) - assert.NoError(t, err) - assert.True(t, found) - assert.Equal(t, 4, version) - } - - // Fetch all partitions and compare. - { - parts, found, err := copy.Partitions(ctx, qtid1) - assert.NoError(t, err) - assert.True(t, found) - assert.Equal(t, partitions, parts) - } - - // Fetch all fieldVersions and compare. - { - flds, found, err := copy.Fields(ctx, qtid1) - assert.NoError(t, err) - assert.True(t, found) - assert.Equal(t, fieldVersions, flds) - } - }) -} - -// newQualifiedTableIDs is a test helper function which generates a slice of n -// qtid. The entries in the slice will be ordered by TableID. -func newQualifiedTableIDs(t *testing.T, qual dax.TableQualifier, n int) []dax.QualifiedTableID { - t.Helper() - - qtids := make([]dax.QualifiedTableID, n) - for i := range qtids { - tbl := dax.NewTable("testvstore") - tbl.CreateID() - qtids[i] = dax.NewQualifiedTableID( - qual, - tbl.ID, - ) - } - - // sort the qtids by ID - sort.Slice(qtids, func(i, j int) bool { - return qtids[i].ID < qtids[j].ID - }) - - return qtids -} diff --git a/dax/directive.go b/dax/directive.go index f5504864d..a0eccad26 100644 --- a/dax/directive.go +++ b/dax/directive.go @@ -1,5 +1,7 @@ package dax +import "context" + // Directive contains the instructions, sent from MDS, which a compute node is // to follow. A Directive is typically JSON-encoded and POSTed to a compute // node's `/directive` endpoint. @@ -19,6 +21,10 @@ type Directive struct { Version uint64 `json:"version"` } +type DirectiveVersion interface { + Increment(ctx context.Context, delta uint64) (uint64, error) +} + // DirectiveMethod is used to tell the compute node how it should handle the // Directive. type DirectiveMethod string @@ -57,9 +63,9 @@ func (d *Directive) Table(qtid QualifiedTableID) (*QualifiedTable, error) { // compute node is responsible. It assumes that the Directive does not contain // more than one ComputeRole for the same table; in that case, we would need to // return the union of Shards. -func (d *Directive) ComputeShards(tbl TableKey) VersionedShards { +func (d *Directive) ComputeShards(tbl TableKey) ShardNums { if d == nil || d.ComputeRoles == nil { - return VersionedShards{} + return nil } for _, cr := range d.ComputeRoles { @@ -68,14 +74,14 @@ func (d *Directive) ComputeShards(tbl TableKey) VersionedShards { } } - return VersionedShards{} + return nil } // ComputeShardsMap returns a map of table to shards. It assumes that the // Directive does not contain more than one ComputeRole for the same table; in // that case, we would need to return the union of Shards. -func (d *Directive) ComputeShardsMap() map[TableKey]VersionedShards { - m := make(map[TableKey]VersionedShards) +func (d *Directive) ComputeShardsMap() map[TableKey]ShardNums { + m := make(map[TableKey]ShardNums) if d == nil || d.ComputeRoles == nil { return m } @@ -91,9 +97,9 @@ func (d *Directive) ComputeShardsMap() map[TableKey]VersionedShards { // which this translate node is responsible. It assumes that the Directive does // not contain more than one TranslateRole for the same table; in that case, we // would need to return the union of Shards. -func (d *Directive) TranslatePartitions(tbl TableKey) VersionedPartitions { +func (d *Directive) TranslatePartitions(tbl TableKey) PartitionNums { if d == nil || d.TranslateRoles == nil { - return VersionedPartitions{} + return PartitionNums{} } for _, tr := range d.TranslateRoles { @@ -101,14 +107,14 @@ func (d *Directive) TranslatePartitions(tbl TableKey) VersionedPartitions { return tr.Partitions } } - return VersionedPartitions{} + return PartitionNums{} } // TranslatePartitionsMap returns a map of table to partitions. It assumes that // the Directive does not contain more than one TranslateRole for the same // table; in that case, we would need to return the union of Partitions. -func (d *Directive) TranslatePartitionsMap() map[TableKey]VersionedPartitions { - m := make(map[TableKey]VersionedPartitions) +func (d *Directive) TranslatePartitionsMap() map[TableKey]PartitionNums { + m := make(map[TableKey]PartitionNums) if d == nil || d.TranslateRoles == nil { return m } @@ -129,8 +135,8 @@ func (d *Directive) TranslatePartitionsMap() map[TableKey]VersionedPartitions { // TranslateFieldsMap returns a map of table to fields. It assumes that // the Directive does not contain more than one TranslateRole for the same // table; in that case, we would need to return the union of FieldValues. -func (d *Directive) TranslateFieldsMap() map[TableKey]VersionedFields { - m := make(map[TableKey]VersionedFields) +func (d *Directive) TranslateFieldsMap() map[TableKey][]FieldName { + m := make(map[TableKey][]FieldName) if d == nil || d.TranslateRoles == nil { return m } diff --git a/dax/inmem/inmem.go b/dax/inmem/inmem.go deleted file mode 100644 index 9a2663214..000000000 --- a/dax/inmem/inmem.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package inmem contains the in-memory implementation of the dax interfaces. -package inmem diff --git a/dax/inmem/versionstore.go b/dax/inmem/versionstore.go deleted file mode 100644 index a6071344a..000000000 --- a/dax/inmem/versionstore.go +++ /dev/null @@ -1,430 +0,0 @@ -package inmem - -import ( - "context" - "sort" - "sync" - - "github.com/featurebasedb/featurebase/v3/dax" - "github.com/featurebasedb/featurebase/v3/errors" -) - -// Ensure type implements interface. -var _ dax.VersionStore = (*VersionStore)(nil) - -// VersionStore manages all version info for shard, table keys, and field keys. -type VersionStore struct { - mu sync.RWMutex - - // shards is a map of all shards, by table, by shard number, known to - // contain data. - shards map[dax.TableQualifierKey]map[dax.TableID]map[dax.ShardNum]dax.VersionedShard - - // tableKeys is a map of all partitions, by table, by partition number, - // known to contain key data. - tableKeys map[dax.TableQualifierKey]map[dax.TableID]map[dax.PartitionNum]int - - // fieldKeys is a map of all fields, by table, known to contain key data. - fieldKeys map[dax.TableQualifierKey]map[dax.TableID]map[dax.FieldName]int -} - -// NewVersionStore returns a new instance of VersionStore with default values. -func NewVersionStore() *VersionStore { - return &VersionStore{ - shards: make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.ShardNum]dax.VersionedShard), - tableKeys: make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.PartitionNum]int), - fieldKeys: make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.FieldName]int), - } -} - -// AddTable adds a table to be managed by VersionStore. -func (s *VersionStore) AddTable(ctx context.Context, qtid dax.QualifiedTableID) error { - s.mu.Lock() - defer s.mu.Unlock() - - // This check is clunky; three maps contain the table, but we only check for - // existence in one of them. It also seems weird to check all three, because - // if we get in a state where one of the maps doesn't contain a table that - // the other maps do contain, the state of the data is in question. - if _, found := s.shards[qtid.TableQualifier.Key()][qtid.ID]; found { - return dax.NewErrTableIDExists(qtid) - } - - // Initialize the maps in case VersionStore wasn't created with NewVersionStore(). - if s.shards == nil { - s.shards = make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.ShardNum]dax.VersionedShard) - } - if s.tableKeys == nil { - s.tableKeys = make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.PartitionNum]int) - } - if s.fieldKeys == nil { - s.fieldKeys = make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.FieldName]int) - } - - // shards. - if _, ok := s.shards[qtid.TableQualifier.Key()]; !ok { - s.shards[qtid.TableQualifier.Key()] = make(map[dax.TableID]map[dax.ShardNum]dax.VersionedShard, 0) - } - if _, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]; !ok { - s.shards[qtid.TableQualifier.Key()][qtid.ID] = make(map[dax.ShardNum]dax.VersionedShard, 0) - } - - // tableKeys. - if _, ok := s.tableKeys[qtid.TableQualifier.Key()]; !ok { - s.tableKeys[qtid.TableQualifier.Key()] = make(map[dax.TableID]map[dax.PartitionNum]int, 0) - } - if _, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]; !ok { - s.tableKeys[qtid.TableQualifier.Key()][qtid.ID] = make(map[dax.PartitionNum]int, 0) - } - - // fieldKeys. - if _, ok := s.fieldKeys[qtid.TableQualifier.Key()]; !ok { - s.fieldKeys[qtid.TableQualifier.Key()] = make(map[dax.TableID]map[dax.FieldName]int, 0) - } - if _, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]; !ok { - s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID] = make(map[dax.FieldName]int, 0) - } - - return nil -} - -// RemoveTable removes the given table. An error will be returned if the table -// does not exist. -func (s *VersionStore) RemoveTable(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedShards, dax.VersionedPartitions, error) { - s.mu.Lock() - defer s.mu.Unlock() - - var foundTable bool - var shards dax.VersionedShards - var partitions dax.VersionedPartitions - var err error - - // Remove shards for table. - if s.shards != nil { - if _, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]; ok { - foundTable = true - - // Get the shards to return before deleting from map. - shards, _, err = s.shardSlice(qtid) - if err != nil { - return nil, nil, errors.Wrapf(err, "getting shard slice: %s", qtid) - } - - // Remove the shards. - delete(s.shards[qtid.TableQualifier.Key()], qtid.ID) - } - } - - // Remove tableKeys for table. - if s.tableKeys != nil { - if _, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]; ok { - foundTable = true - - // Get the partitions to return before deleting from map. - partitions, _, err = s.partitionSlice(qtid) - if err != nil { - return nil, nil, errors.Wrapf(err, "getting partition slice: %s", qtid) - } - - // Remove the tableKeys. - delete(s.tableKeys[qtid.TableQualifier.Key()], qtid.ID) - } - } - - // Remove fieldKeys for table. - if s.fieldKeys != nil { - if _, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]; ok { - foundTable = true - - // Remove the fieldKeys. - delete(s.fieldKeys[qtid.TableQualifier.Key()], qtid.ID) - } - } - - if !foundTable { - return nil, nil, dax.NewErrTableIDDoesNotExist(qtid) - } - - return shards, partitions, nil -} - -// AddShards adds new shards to be managed by VersionStore. It returns the -// number of shards added or an error. -func (s *VersionStore) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.VersionedShard) error { - s.mu.Lock() - defer s.mu.Unlock() - - sh, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID] - if !ok { - return dax.NewErrTableIDDoesNotExist(qtid) - } - - var n int - - for _, shard := range shards { - if _, ok := sh[shard.Num]; !ok { - n++ // TODO: this isn't considering a shard that exists, but the version changes. - } - sh[shard.Num] = shard - } - - return nil -} - -// Shards returns the list of shards available for the give table. It returns -// false if the table does not exist. -func (s *VersionStore) Shards(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedShards, bool, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - return s.shardSlice(qtid) -} - -// shardSlice is an unprotected version of Shards(). -func (s *VersionStore) shardSlice(qtid dax.QualifiedTableID) (dax.VersionedShards, bool, error) { - if s.shards == nil { - return nil, false, nil - } - - if shardNumMap, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]; ok { - rtn := make(dax.VersionedShards, 0, len(shardNumMap)) - for _, shard := range shardNumMap { - rtn = append(rtn, shard) - } - sort.Sort(rtn) - return rtn, true, nil - } - - return nil, false, nil -} - -// ShardVersion return the current version for the given table/shardNum. -// If a version is not being tracked, it returns a bool value of false. -func (s *VersionStore) ShardVersion(ctx context.Context, qtid dax.QualifiedTableID, shardNum dax.ShardNum) (int, bool, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - t, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID] - if !ok { - return -1, false, nil - } - - v, ok := t[shardNum] - if !ok { - return -1, false, nil - } - return v.Version, true, nil -} - -func (s *VersionStore) ShardTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - qual.Key() - - tableIDs := make(dax.TableIDs, 0, len(s.shards[qual.Key()])) - - for tableID := range s.shards[qual.Key()] { - tableIDs = append(tableIDs, tableID) - } - - return tableIDs, nil -} - -// AddPartitions adds new partitions to be managed by VersionStore. It returns -// the number of partitions added or an error. -func (s *VersionStore) AddPartitions(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.VersionedPartition) error { - s.mu.Lock() - defer s.mu.Unlock() - - tk, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID] - if !ok { - return dax.NewErrTableIDDoesNotExist(qtid) - } - - for _, partition := range partitions { - tk[partition.Num] = partition.Version - } - - return nil -} - -// Partitions returns the list of partitions available for the give table. It -// returns false if the table does not exist. -func (s *VersionStore) Partitions(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedPartitions, bool, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - return s.partitionSlice(qtid) -} - -// partitionSlice is an unprotected version of Partitions(). -func (s *VersionStore) partitionSlice(qtid dax.QualifiedTableID) (dax.VersionedPartitions, bool, error) { - if s.tableKeys == nil { - return nil, false, nil - } - - if partitionNumMap, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]; ok { - rtn := make(dax.VersionedPartitions, 0, len(partitionNumMap)) - for partitionNum, version := range partitionNumMap { - rtn = append(rtn, dax.NewVersionedPartition(partitionNum, version)) - } - sort.Sort(rtn) - return rtn, true, nil - } - - return nil, false, nil -} - -// PartitionVersion return the current version for the given table/partitionNum. -// If a version is not being tracked, it returns a bool value of false. -func (s *VersionStore) PartitionVersion(ctx context.Context, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) (int, bool, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - t, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID] - if !ok { - return -1, false, nil - } - - v, ok := t[partitionNum] - if !ok { - return -1, false, nil - } - return v, true, nil -} - -func (s *VersionStore) PartitionTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - tableIDs := make(dax.TableIDs, 0, len(s.tableKeys[qual.Key()])) - - for tableName := range s.tableKeys[qual.Key()] { - tableIDs = append(tableIDs, tableName) - } - - return tableIDs, nil -} - -// AddFields adds new fields to be managed by VersionStore. It returns the -// number of fields added or an error. -func (s *VersionStore) AddFields(ctx context.Context, qtid dax.QualifiedTableID, fields ...dax.VersionedField) error { - s.mu.Lock() - defer s.mu.Unlock() - - fk, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID] - if !ok { - return dax.NewErrTableIDDoesNotExist(qtid) - } - - for _, field := range fields { - fk[field.Name] = field.Version - } - - return nil -} - -// Fields returns the list of fields available for the give table. It returns -// false if the table does not exist. -func (s *VersionStore) Fields(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedFields, bool, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - return s.fieldSlice(qtid) -} - -// fieldSlice is an unprotected version of Fields(). -func (s *VersionStore) fieldSlice(qtid dax.QualifiedTableID) (dax.VersionedFields, bool, error) { - if s.fieldKeys == nil { - return nil, false, nil - } - - if fieldNameMap, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]; ok { - rtn := make(dax.VersionedFields, 0, len(fieldNameMap)) - for fieldName, version := range fieldNameMap { - rtn = append(rtn, dax.NewVersionedField(fieldName, version)) - } - sort.Sort(rtn) - return rtn, true, nil - } - - return nil, false, nil -} - -// FieldVersion return the current version for the given table/field. -// If a version is not being tracked, it returns a bool value of false. -func (s *VersionStore) FieldVersion(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName) (int, bool, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - t, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID] - if !ok { - return -1, false, nil - } - - v, ok := t[field] - if !ok { - return -1, false, nil - } - return v, true, nil -} - -func (s *VersionStore) FieldTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - tableIDs := make(dax.TableIDs, 0, len(s.fieldKeys[qual.Key()])) - - for tableID := range s.fieldKeys[qual.Key()] { - tableIDs = append(tableIDs, tableID) - } - - return tableIDs, nil -} - -// Copy returns a new copy of VersionStore. -func (s *VersionStore) Copy(ctx context.Context) (dax.VersionStore, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - new := NewVersionStore() - - // shards. - for qkey, tableIDs := range s.shards { - for tableID, shards := range tableIDs { - qual := dax.NewTableQualifier(qkey.OrganizationID(), qkey.DatabaseID()) - qtid := dax.NewQualifiedTableID(qual, tableID) - _ = new.AddTable(ctx, qtid) - for shardNum, shard := range shards { - new.shards[qual.Key()][tableID][shardNum] = shard - } - } - } - - // tableKeys. - for qkey, tableIDs := range s.tableKeys { - for tableID, partitions := range tableIDs { - qual := dax.NewTableQualifier(qkey.OrganizationID(), qkey.DatabaseID()) - qtid := dax.NewQualifiedTableID(qual, tableID) - _ = new.AddTable(ctx, qtid) - for partitionNum, version := range partitions { - new.tableKeys[qual.Key()][tableID][partitionNum] = version - } - } - } - - // fieldKeys. - for qkey, tableIDs := range s.fieldKeys { - for tableID, fields := range tableIDs { - qual := dax.NewTableQualifier(qkey.OrganizationID(), qkey.DatabaseID()) - qtid := dax.NewQualifiedTableID(qual, tableID) - _ = new.AddTable(ctx, qtid) - for field, version := range fields { - new.fieldKeys[qual.Key()][tableID][field] = version - } - } - } - - return new, nil -} diff --git a/dax/inmem/versionstore_test.go b/dax/inmem/versionstore_test.go deleted file mode 100644 index 3ccf39967..000000000 --- a/dax/inmem/versionstore_test.go +++ /dev/null @@ -1,166 +0,0 @@ -package inmem_test - -import ( - "context" - "testing" - - "github.com/featurebasedb/featurebase/v3/dax" - "github.com/featurebasedb/featurebase/v3/dax/inmem" - "github.com/featurebasedb/featurebase/v3/errors" - "github.com/stretchr/testify/assert" -) - -func TestVersionStore(t *testing.T) { - orgID := dax.OrganizationID("acme") - dbID := dax.DatabaseID("db1") - - tableID := dax.TableID("0000000000000001") - - qual := dax.NewTableQualifier(orgID, dbID) - qtid := dax.NewQualifiedTableID(qual, tableID) - - invalidQtid := dax.NewQualifiedTableID(qual, dax.TableID("0000000000000000")) - - ctx := context.Background() - - // Ensure that when using a Schemar not initiated with NewSchemar, the error - // handling works as expected. - t.Run("EmptyVersionStore", func(t *testing.T) { - s := inmem.VersionStore{} - - t.Run("GetShardsInvalid", func(t *testing.T) { - sh, ok, err := s.Shards(ctx, invalidQtid) - assert.NoError(t, err) - assert.False(t, ok) - assert.Nil(t, sh) - }) - - // Add new table. - assert.NoError(t, s.AddTable(ctx, qtid)) - }) - - t.Run("NewVersionStore", func(t *testing.T) { - s := inmem.NewVersionStore() - - // Add new table. - assert.NoError(t, s.AddTable(ctx, qtid)) - - t.Run("AddTableAgain", func(t *testing.T) { - err := s.AddTable(ctx, qtid) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrTableIDExists)) - } - }) - - t.Run("AddShards", func(t *testing.T) { - err := s.AddShards(ctx, invalidQtid, dax.NewVersionedShard(1, 0)) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist)) - } - - { - _, ok, err := s.Shards(ctx, invalidQtid) - assert.NoError(t, err) - assert.False(t, ok) - } - - // Shards is empty if no shards have been added. - { - sh, ok, err := s.Shards(ctx, qtid) - assert.NoError(t, err) - assert.True(t, ok) - assert.Equal(t, sh, dax.VersionedShards{}) - } - - // Add the first set of shards (with a duplicate (8)). - { - err := s.AddShards(ctx, qtid, - dax.NewVersionedShard(8, 0), - dax.NewVersionedShard(9, 0), - dax.NewVersionedShard(8, 0), - dax.NewVersionedShard(10, 0), - ) - assert.NoError(t, err) - } - - { - sh, ok, err := s.Shards(ctx, qtid) - assert.NoError(t, err) - assert.True(t, ok) - assert.Equal(t, dax.VersionedShards{ - dax.NewVersionedShard(8, 0), - dax.NewVersionedShard(9, 0), - dax.NewVersionedShard(10, 0), - }, sh) - } - - // Add another set of shards (with one duplicate (11) and one - // existing (10)). - { - err := s.AddShards(ctx, qtid, - dax.NewVersionedShard(10, 0), - dax.NewVersionedShard(11, 0), - dax.NewVersionedShard(12, 0), - dax.NewVersionedShard(11, 0), - ) - assert.NoError(t, err) - } - - { - sh, ok, err := s.Shards(ctx, qtid) - assert.NoError(t, err) - assert.True(t, ok) - assert.Equal(t, dax.VersionedShards{ - dax.NewVersionedShard(8, 0), - dax.NewVersionedShard(9, 0), - dax.NewVersionedShard(10, 0), - dax.NewVersionedShard(11, 0), - dax.NewVersionedShard(12, 0), - }, sh) - } - }) - - t.Run("RemoveTable", func(t *testing.T) { - shards, partitions, err := s.RemoveTable(ctx, qtid) - assert.NoError(t, err) - assert.Equal(t, dax.VersionedPartitions{}, partitions) - assert.Equal(t, dax.VersionedShards{ - dax.NewVersionedShard(8, 0), - dax.NewVersionedShard(9, 0), - dax.NewVersionedShard(10, 0), - dax.NewVersionedShard(11, 0), - dax.NewVersionedShard(12, 0), - }, shards) - - // Make sure the table was removed. - shards, ok, err := s.Shards(ctx, qtid) - assert.NoError(t, err) - assert.False(t, ok) - assert.Nil(t, shards) - }) - }) - - t.Run("ErrorConditions", func(t *testing.T) { - t.Run("JustSchemar", func(t *testing.T) { - s := inmem.VersionStore{} - - shards, partitions, err := s.RemoveTable(ctx, qtid) - assert.Nil(t, shards) - assert.Nil(t, partitions) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist)) - } - }) - - t.Run("NewSchemar", func(t *testing.T) { - s := inmem.NewVersionStore() - - shards, partitions, err := s.RemoveTable(ctx, qtid) - assert.Nil(t, shards) - assert.Nil(t, partitions) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist)) - } - }) - }) -} diff --git a/dax/mds/controller/balancer.go b/dax/mds/controller/balancer.go index 0736eddb4..a6402c6fb 100644 --- a/dax/mds/controller/balancer.go +++ b/dax/mds/controller/balancer.go @@ -28,6 +28,11 @@ type Balancer interface { // which are not assigned to any worker, that means the query // would return incomplete data, so we want to error. WorkersForJobPrefix(ctx context.Context, prefix string) ([]dax.WorkerInfo, error) + + // RemoveJobs is for e.g. when dropping a table remove all jobs + // associated with that table without needing to look up in + // advance which shards or partitions are actually present. + RemoveJobs(ctx context.Context, prefix string) ([]dax.WorkerDiff, error) } // Ensure type implements interface. @@ -67,3 +72,7 @@ func (b *NopBalancer) WorkersForJobs(ctx context.Context, jobs []dax.Job) ([]dax func (b *NopBalancer) WorkersForJobPrefix(ctx context.Context, prefix string) ([]dax.WorkerInfo, error) { return []dax.WorkerInfo{}, nil } + +func (b *NopBalancer) RemoveJobs(ctx context.Context, prefix string) ([]dax.WorkerDiff, error) { + return nil, nil +} diff --git a/dax/mds/controller/controller.go b/dax/mds/controller/controller.go index 23b38f563..529537db7 100644 --- a/dax/mds/controller/controller.go +++ b/dax/mds/controller/controller.go @@ -22,9 +22,6 @@ type Controller struct { // initialization. mu sync.RWMutex - // versionStore - versionStore dax.VersionStore - // Schemar used by the controller to get table information. The controller // should NOT call Schemar methods which modify data. Schema mutations are // made outside of the controller (at this point that happens in MDS). @@ -84,11 +81,6 @@ func New(cfg Config) *Controller { switch cfg.StorageMethod { case "boltdb": - if err := cfg.BoltDB.InitializeBuckets(boltdb.VersionStoreBuckets...); err != nil { - c.logger.Panicf("initializing version store buckets: %v", err) - } - c.versionStore = boltdb.NewVersionStore(cfg.BoltDB, c.logger) - if err := cfg.BoltDB.InitializeBuckets(boltdb.NodeServiceBuckets...); err != nil { c.logger.Panicf("initializing node service buckets: %v", err) } @@ -520,12 +512,6 @@ func (c *Controller) nodesTranslateReadOrWrite(ctx context.Context, role *dax.Tr for _, diff := range diffs { workerSet.Add(dax.Address(diff.WorkerID)) } - - // Initialize the partition version to 0. - qtid := j.table().QualifiedTableID() - if err := c.versionStore.AddPartitions(ctx, qtid, dax.NewVersionedPartition(j.partitionNum(), 0)); err != nil { - return nil, false, NewErrInternal(err.Error()) - } } // Convert the slice of addresses into a slice of addressMethod containing @@ -546,26 +532,14 @@ func (c *Controller) nodesTranslateReadOrWrite(ctx context.Context, role *dax.Tr for _, worker := range workers { // covert worker.Jobs []string to map[string][]Partition - translateMap := make(map[dax.TableKey]dax.VersionedPartitions) + translateMap := make(map[dax.TableKey]dax.PartitionNums) for _, job := range worker.Jobs { j, err := decodePartition(job) if err != nil { return nil, false, NewErrInternal(err.Error()) } - // Get the partition version from the local versionStore. - tkey := j.table() - qtid := tkey.QualifiedTableID() - partitionVersion, found, err := c.versionStore.PartitionVersion(ctx, qtid, j.partitionNum()) - if err != nil { - return nil, false, err - } else if !found { - return nil, false, NewErrInternal("partition version not found in cache") - } - - translateMap[tkey] = append(translateMap[tkey], - dax.NewVersionedPartition(j.partitionNum(), partitionVersion), - ) + translateMap[j.table()] = append(translateMap[j.table()], j.partitionNum()) } for table, partitions := range translateMap { @@ -707,12 +681,6 @@ func (c *Controller) nodesComputeReadOrWrite(ctx context.Context, role *dax.Comp for _, diff := range diffs { workerSet.Add(dax.Address(diff.WorkerID)) } - - // Initialize the shard version to 0. - qtid := j.table().QualifiedTableID() - if err := c.versionStore.AddShards(ctx, qtid, dax.NewVersionedShard(j.shardNum(), 0)); err != nil { - return nil, false, NewErrInternal(err.Error()) - } } // Convert the slice of addresses into a slice of addressMethod containing @@ -739,26 +707,14 @@ func (c *Controller) workersToAssignedNodes(ctx context.Context, workers []dax.W nodes := []dax.AssignedNode{} for _, worker := range workers { // convert worker.Jobs []string to map[TableName][]Shard - computeMap := make(map[dax.TableKey]dax.VersionedShards) + computeMap := make(map[dax.TableKey]dax.ShardNums) for _, job := range worker.Jobs { j, err := decodeShard(job) if err != nil { return nil, NewErrInternal(err.Error()) } - // Get the shard version from the local versionStore. - tkey := j.table() - qtid := tkey.QualifiedTableID() - shardVersion, found, err := c.versionStore.ShardVersion(ctx, qtid, j.shardNum()) - if err != nil { - return nil, err - } else if !found { - return nil, NewErrInternal("shard version not found in cache") - } - - computeMap[tkey] = append(computeMap[tkey], - dax.NewVersionedShard(j.shardNum(), shardVersion), - ) + computeMap[j.table()] = append(computeMap[j.table()], j.shardNum()) } for table, shards := range computeMap { @@ -783,31 +739,6 @@ func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) c.mu.Lock() defer c.mu.Unlock() - qtid := qtbl.QualifiedID() - - // Add the table to the versionStore. - if err := c.versionStore.AddTable(ctx, qtid); err != nil { - return errors.Wrapf(err, "adding table: %s", qtid) - } - - // Add fields which have string keys to the local versionStore. - fieldVersions := make(dax.VersionedFields, 0) - for _, field := range qtbl.Fields { - if !field.StringKeys() { - continue - } - - fieldVersions = append(fieldVersions, dax.VersionedField{ - Name: field.Name, - Version: 0, - }) - } - if len(fieldVersions) > 0 { - if err := c.versionStore.AddFields(ctx, qtid, fieldVersions...); err != nil { - return errors.Wrapf(err, "adding fields: %s, %v", qtid, fieldVersions) - } - } - // If the table is keyed, add partitions to the balancer. if qtbl.StringKeys() { // workerSet maintains the set of workers which have a job assignment change @@ -815,15 +746,9 @@ func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) workerSet := NewAddressSet() // Generate the list of partitionsToAdd to be added. - partitionsToAdd := make(dax.VersionedPartitions, qtbl.PartitionN) + partitionsToAdd := make(dax.PartitionNums, qtbl.PartitionN) for partitionNum := 0; partitionNum < qtbl.PartitionN; partitionNum++ { - partitionsToAdd[partitionNum] = dax.NewVersionedPartition(dax.PartitionNum(partitionNum), 0) - } - - // Add partitions to versionStore. Version is intentionally set to 0 - // here as this is the initial instance of the partition. - if err := c.versionStore.AddPartitions(ctx, qtid, partitionsToAdd...); err != nil { - return NewErrInternal(err.Error()) + partitionsToAdd[partitionNum] = dax.PartitionNum(partitionNum) } stringers := make([]fmt.Stringer, 0, len(partitionsToAdd)) @@ -858,13 +783,7 @@ func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) // and therefore need to be sent an updated Directive. workerSet := NewAddressSet() - p := dax.NewVersionedPartition(0, 0) - - // Add partition 0 to versionStore. Version is intentionally set to 0 - // here as this is the initial instance of the partition. - if err := c.versionStore.AddPartitions(ctx, qtid, p); err != nil { - return NewErrInternal(err.Error()) - } + p := dax.PartitionNum(0) // We don't currently use the returned diff, other than to determine // which worker was affected, because we send the full Directive @@ -900,39 +819,24 @@ func (c *Controller) DropTable(ctx context.Context, qtid dax.QualifiedTableID) e return errors.Wrapf(err, "table not in schemar: %s", qtid) } - // Remove the table from the versionStore. - // Since the schemar should be the system of record for the existence of a - // table, if the versionStore is not aware of the table, we just log it and - // continue. - shards, partitions, err := c.versionStore.RemoveTable(ctx, qtid) - if err != nil { - return errors.Wrapf(err, "removing table: %s", qtid) - } - // workerSet maintains the set of workers which have a job assignment change // and therefore need to be sent an updated Directive. workerSet := NewAddressSet() - // Remove shards. - for _, s := range shards { - diffs, err := c.ComputeBalancer.RemoveJob(ctx, shard(qtid.Key(), s)) - if err != nil { - return errors.Wrap(err, "removing job") - } - for _, diff := range diffs { - workerSet.Add(dax.Address(diff.WorkerID)) - } + diffs, err := c.ComputeBalancer.RemoveJobs(ctx, string(qtid.Key())) + if err != nil { + return errors.Wrap(err, "removing jobs") + } + for _, diff := range diffs { + workerSet.Add(dax.Address(diff.WorkerID)) } - // Remove partitions. - for _, p := range partitions { - diffs, err := c.TranslateBalancer.RemoveJob(ctx, partition(qtid.Key(), p)) - if err != nil { - return errors.Wrap(err, "removing job") - } - for _, diff := range diffs { - workerSet.Add(dax.Address(diff.WorkerID)) - } + diffs, err = c.TranslateBalancer.RemoveJobs(ctx, string(qtid.Key())) + if err != nil { + return errors.Wrap(err, "removing job") + } + for _, diff := range diffs { + workerSet.Add(dax.Address(diff.WorkerID)) } // Convert the slice of addresses into a slice of addressMethod containing @@ -966,15 +870,10 @@ func (c *Controller) Tables(ctx context.Context, qual dax.TableQualifier, ids .. // AddShards registers the table/shard combinations with the controller and // sends the necessary directive. -func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.VersionedShard) error { +func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) error { c.mu.Lock() defer c.mu.Unlock() - // Add shards to versionStore. - if err := c.versionStore.AddShards(ctx, qtid, shards...); err != nil { - return errors.Wrapf(err, "adding shards: %s, %v", qtid, shards) - } - // workerSet maintains the set of workers which have a job assignment change // and therefore need to be sent an updated Directive. workerSet := NewAddressSet() @@ -990,11 +889,6 @@ func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, s for _, diff := range diffs { workerSet.Add(dax.Address(diff.WorkerID)) } - - // Initialize the shard version to 0. - if err := c.versionStore.AddShards(ctx, qtid, dax.NewVersionedShard(s.Num, 0)); err != nil { - return NewErrInternal(err.Error()) - } } // Convert the slice of addresses into a slice of addressMethod containing @@ -1010,7 +904,7 @@ func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, s // RemoveShards deregisters the table/shard combinations with the controller and // sends the necessary directives. -func (c *Controller) RemoveShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.VersionedShard) error { +func (c *Controller) RemoveShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) error { c.mu.Lock() defer c.mu.Unlock() @@ -1050,7 +944,7 @@ func (c *Controller) sendDirectives(ctx context.Context, addrs ...addressMethod) return nil } - directives, err := c.buildDirectives(ctx, addrs, c.versionStore) + directives, err := c.buildDirectives(ctx, addrs) if err != nil { return errors.Wrap(err, "building directives") } @@ -1120,7 +1014,7 @@ func applyAddressMethod(addrs []dax.Address, method dax.DirectiveMethod) []addre // buildDirectives builds a list of directives for the given addrs (i.e. nodes) // using information (i.e. current state) from the balancers. -func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod, versionStore dax.VersionStore) ([]*dax.Directive, error) { +func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod) ([]*dax.Directive, error) { directives := make([]*dax.Directive, len(addrs)) for i, addressMethod := range addrs { @@ -1141,12 +1035,12 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod, // computeMap maps a table to a list of shards for that table. We need // to aggregate them here because the list of jobs from WorkerState() // can contain a mixture of table/shards. - computeMap := make(map[dax.TableKey][]dax.VersionedShard) + computeMap := make(map[dax.TableKey][]dax.ShardNum) // translateMap maps a table to a list of partitions for that table. We // need to aggregate them here because the list of jobs from // WorkerState() can contain a mixture of table/partitions. - translateMap := make(map[dax.TableKey]dax.VersionedPartitions) + translateMap := make(map[dax.TableKey][]dax.PartitionNum) // tableSet maintains the set of tables which have a job assignment // change and therefore need to be included in the Directive schema. @@ -1174,24 +1068,8 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod, return nil, errors.Wrapf(err, "decoding shard job: %s", job) } - // The Shard object decoded from the balancer doesn't - // contain a valid version (because the balancer - // intentionally does not store version information). Here, - // we get the current shard version from the controller's - // cache (i.e. versionStore) and inject that into the Shard - // sent in the directive. tkey := j.table() - qtid := tkey.QualifiedTableID() - shardVersion, found, err := versionStore.ShardVersion(ctx, qtid, j.shardNum()) - if err != nil { - return nil, errors.Wrapf(err, "getting shard version: %s, %d", qtid, j.shardNum()) - } else if !found { - return nil, NewErrInternal("shard version not found in cache") - } - - computeMap[tkey] = append(computeMap[tkey], - dax.NewVersionedShard(j.shardNum(), shardVersion), - ) + computeMap[tkey] = append(computeMap[tkey], j.shardNum()) tableSet.Add(tkey) } case dax.RoleTypeTranslate: @@ -1209,24 +1087,8 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod, ownsPartition0[j.table()] = struct{}{} } - // The Partition object decoded from the balancer doesn't - // contain a valid version (because the balancer - // intentionally does not store version information). Here, - // we get the current partition version from the - // controller's cache (i.e. versionStore) and inject that - // into the Partition sent in the directive. tkey := j.table() - qtid := tkey.QualifiedTableID() - partitionVersion, found, err := versionStore.PartitionVersion(ctx, qtid, j.partitionNum()) - if err != nil { - return nil, errors.Wrapf(err, "getting partition version: %s, %d", qtid, j.partitionNum()) - } else if !found { - return nil, NewErrInternal("partition version not found in cache") - } - - translateMap[tkey] = append(translateMap[tkey], - dax.NewVersionedPartition(j.partitionNum(), partitionVersion), - ) + translateMap[tkey] = append(translateMap[tkey], j.partitionNum()) tableSet.Add(tkey) } } @@ -1237,7 +1099,7 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod, // Because these were encoded as strings in the balancer and may be // out of order numerically, sort them as integers. //sort.Slice(v, func(i, j int) bool { return v[i] < v[j] }) - sort.Sort(dax.VersionedShards(v)) + sort.Sort(dax.ShardNums(v)) d.ComputeRoles = append(d.ComputeRoles, dax.ComputeRole{ TableKey: k, @@ -1249,7 +1111,7 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod, for k, v := range translateMap { // Because these were encoded as strings in the balancer and may be // out of order numerically, sort them as integers. - sort.Sort(v) + sort.Sort(dax.PartitionNums(v)) d.TranslateRoles = append(d.TranslateRoles, dax.TranslateRole{ TableKey: k, @@ -1271,7 +1133,7 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod, return nil, errors.Wrapf(err, "getting table: %s", tkey) } - fieldVersions := make(dax.VersionedFields, 0) + fieldNames := make([]dax.FieldName, 0) for _, field := range table.Fields { if !field.StringKeys() { continue @@ -1282,26 +1144,16 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod, continue } - fieldVersion, found, err := versionStore.FieldVersion(ctx, qtid, field.Name) - if err != nil { - return nil, errors.Wrapf(err, "getting field version: %s, %s", qtid, field) - } else if !found { - return nil, NewErrInternal("field version not found in cache") - } - - fieldVersions = append(fieldVersions, dax.VersionedField{ - Name: field.Name, - Version: fieldVersion, - }) + fieldNames = append(fieldNames, field.Name) } - if len(fieldVersions) == 0 { + if len(fieldNames) == 0 { continue } d.TranslateRoles = append(d.TranslateRoles, dax.TranslateRole{ TableKey: tkey, - Fields: fieldVersions, + Fields: fieldNames, }) tableSet.Add(tkey) @@ -1355,48 +1207,50 @@ func (c *Controller) InitializePoller(ctx context.Context) error { } // SnapshotTable snapshots a table. +// TODO(jaffee): do we need to re-implement this w/o version store? Or can we just remove entirely? func (c *Controller) SnapshotTable(ctx context.Context, qtid dax.QualifiedTableID) error { - shards, ok, err := c.versionStore.Shards(ctx, qtid) - if err != nil { - return errors.Wrap(err, "getting shards from version store") - } else if !ok { - return errors.New(errors.ErrUncoded, "got false back from versionStore.Shards") - } - - for _, shard := range shards { - if err := c.SnapshotShardData(ctx, qtid, shard.Num); err != nil { - return errors.Wrapf(err, "snapshotting shard data: qtid: %s, shard: %d", qtid, shard.Num) - } - } - - partitions, ok, err := c.versionStore.Partitions(ctx, qtid) - if err != nil { - return errors.Wrap(err, "getting partitions from version store") - } else if !ok { - return errors.New(errors.ErrUncoded, "got false back from versionStore.Partitions") - } - - for _, part := range partitions { - if err := c.SnapshotTableKeys(ctx, qtid, part.Num); err != nil { - return errors.Wrapf(err, "snapshotting table keys: qtid: %s, partition: %d", qtid, part.Num) - } - } - - fields, ok, err := c.versionStore.Fields(ctx, qtid) - if err != nil { - return errors.Wrap(err, "getting fields from version store") - } else if !ok { - return errors.New(errors.ErrUncoded, "got false back from versionStore.Fields") - } - - for _, fld := range fields { - if fld.Name != "_id" { - if err := c.SnapshotFieldKeys(ctx, qtid, fld.Name); err != nil { - return errors.Wrapf(err, "snapshotting field keys: qtid: %s, field: %s", qtid, fld.Name) - } - } - } return nil + // shards, ok, err := c.versionStore.Shards(ctx, qtid) + // if err != nil { + // return errors.Wrap(err, "getting shards from version store") + // } else if !ok { + // return errors.New(errors.ErrUncoded, "got false back from versionStore.Shards") + // } + + // for _, shard := range shards { + // if err := c.SnapshotShardData(ctx, qtid, shard.Num); err != nil { + // return errors.Wrapf(err, "snapshotting shard data: qtid: %s, shard: %d", qtid, shard.Num) + // } + // } + + // partitions, ok, err := c.versionStore.Partitions(ctx, qtid) + // if err != nil { + // return errors.Wrap(err, "getting partitions from version store") + // } else if !ok { + // return errors.New(errors.ErrUncoded, "got false back from versionStore.Partitions") + // } + + // for _, part := range partitions { + // if err := c.SnapshotTableKeys(ctx, qtid, part.Num); err != nil { + // return errors.Wrapf(err, "snapshotting table keys: qtid: %s, partition: %d", qtid, part.Num) + // } + // } + + // fields, ok, err := c.versionStore.Fields(ctx, qtid) + // if err != nil { + // return errors.Wrap(err, "getting fields from version store") + // } else if !ok { + // return errors.New(errors.ErrUncoded, "got false back from versionStore.Fields") + // } + + // for _, fld := range fields { + // if fld.Name != "_id" { + // if err := c.SnapshotFieldKeys(ctx, qtid, fld.Name); err != nil { + // return errors.Wrapf(err, "snapshotting field keys: qtid: %s, field: %s", qtid, fld.Name) + // } + // } + // } + // return nil } // SnapshotShardData forces the compute node responsible for the given shard to @@ -1405,7 +1259,7 @@ func (c *Controller) SnapshotTable(ctx context.Context, qtid dax.QualifiedTableI func (c *Controller) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTableID, shardNum dax.ShardNum) error { // Get the node responsible for the shard. bal := c.ComputeBalancer - job := shard(qtid.Key(), dax.NewVersionedShard(shardNum, -1)) + job := shard(qtid.Key(), shardNum) workers, err := bal.WorkersForJobs(ctx, []dax.Job{dax.Job(job.String())}) if err != nil { return errors.Wrapf(err, "getting workers for jobs: %s", job) @@ -1437,7 +1291,7 @@ func (c *Controller) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTa func (c *Controller) SnapshotTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) error { // Get the node responsible for the partition. bal := c.TranslateBalancer - job := partition(qtid.Key(), dax.NewVersionedPartition(partitionNum, -1)) + job := partition(qtid.Key(), partitionNum) workers, err := bal.WorkersForJobs(ctx, []dax.Job{dax.Job(job.String())}) if err != nil { return errors.Wrapf(err, "getting workers for jobs: %s", job) @@ -1471,7 +1325,7 @@ func (c *Controller) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTa bal := c.TranslateBalancer // Field translation is currently handled by partition 0. partitionNum := dax.PartitionNum(0) - job := partition(qtid.Key(), dax.NewVersionedPartition(partitionNum, -1)) + job := partition(qtid.Key(), partitionNum) workers, err := bal.WorkersForJobs(ctx, []dax.Job{dax.Job(job.String())}) if err != nil { @@ -1503,7 +1357,7 @@ func (c *Controller) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTa func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards dax.ShardNums, isWrite bool) ([]dax.ComputeNode, error) { inRole := &dax.ComputeRole{ TableKey: qtid.Key(), - Shards: dax.NewVersionedShards(shards...), + Shards: shards, } nodes, err := c.Nodes(ctx, inRole, isWrite) @@ -1524,7 +1378,7 @@ func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID computeNodes = append(computeNodes, dax.ComputeNode{ Address: node.Address, Table: role.TableKey, - Shards: role.Shards.Nums(), + Shards: role.Shards, }) } @@ -1534,7 +1388,7 @@ func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions dax.PartitionNums, isWrite bool) ([]dax.TranslateNode, error) { inRole := &dax.TranslateRole{ TableKey: qtid.Key(), - Partitions: dax.NewVersionedPartitions(partitions...), + Partitions: partitions, } nodes, err := c.Nodes(ctx, inRole, isWrite) @@ -1555,7 +1409,7 @@ func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTable translateNodes = append(translateNodes, dax.TranslateNode{ Address: node.Address, Table: role.TableKey, - Partitions: role.Partitions.Nums(), + Partitions: role.Partitions, }) } @@ -1612,37 +1466,19 @@ func (c *Controller) CreateField(ctx context.Context, qtid dax.QualifiedTableID, // and therefore need to be sent an updated Directive. workerSet := NewAddressSet() - // If the field has string keys, add it to the local versionStore. - if fld.StringKeys() { - fieldVersion := dax.VersionedField{ - Name: fld.Name, - Version: 0, - } - if err := c.versionStore.AddFields(ctx, qtid, fieldVersion); err != nil { - return errors.Wrapf(err, "adding fields: %s, %s", qtid, fieldVersion) - } + // Get the worker(s) responsible for partition 0. + job := partition(qtid.Key(), 0).String() + workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)}) + if err != nil { + return errors.Wrapf(err, "getting workers for job: %s", job) } - // Get the worker responsible for partition 0, which handles field key - // translation. Be sure to get the current version. - if v, found, err := c.versionStore.PartitionVersion(ctx, qtid, 0); err != nil { - return errors.Wrapf(err, "getting partition version: %s/0", qtid) - } else if found { - // Get the worker(s) responsible for partition 0. - job := partition(qtid.Key(), dax.VersionedPartition{ - Num: 0, - Version: v, - }).String() - workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)}) - if err != nil { - return errors.Wrapf(err, "getting workers for job: %s", job) - } - - for _, w := range workers { - workerSet.Add(dax.Address(w.ID)) - } + for _, w := range workers { + workerSet.Add(dax.Address(w.ID)) } + // TODO if there isn't already a worker for partition 0, do we need to add it? + // Get the list of workers responsible for shard data for this table. if state, err := c.ComputeBalancer.CurrentState(ctx); err != nil { return errors.Wrap(err, "getting current compute state") @@ -1679,27 +1515,15 @@ func (c *Controller) DropField(ctx context.Context, qtid dax.QualifiedTableID, f // and therefore need to be sent an updated Directive. workerSet := NewAddressSet() - // If the field has string keys, remove it from the local versionStore. - // TODO: implement RemoveField() on VersionStore interface. + // Get the worker(s) responsible for partition 0. + job := partition(qtid.Key(), 0).String() + workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)}) + if err != nil { + return errors.Wrapf(err, "getting workers for job: %s", job) + } - // Get the worker responsible for partition 0, which handles field key - // translation. Be sure to get the current version. - if v, found, err := c.versionStore.PartitionVersion(ctx, qtid, 0); err != nil { - return errors.Wrapf(err, "getting partition version: %s/0", qtid) - } else if found { - // Get the worker(s) responsible for partition 0. - job := partition(qtid.Key(), dax.VersionedPartition{ - Num: 0, - Version: v, - }).String() - workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)}) - if err != nil { - return errors.Wrapf(err, "getting workers for job: %s", job) - } - - for _, w := range workers { - workerSet.Add(dax.Address(w.ID)) - } + for _, w := range workers { + workerSet.Add(dax.Address(w.ID)) } // Get the list of workers responsible for shard data for this table. diff --git a/dax/mds/controller/controller_test.go b/dax/mds/controller/controller_test.go index 07eebfe19..4f36b6d1e 100644 --- a/dax/mds/controller/controller_test.go +++ b/dax/mds/controller/controller_test.go @@ -109,18 +109,17 @@ func TestController(t *testing.T) { assert.Equal(t, exp, director.flush()) // Add the same non-keyed table again. - err := con.CreateTable(ctx, tbl0) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrTableIDExists)) - } + // TODO(jaffee) figure out what this should do + // err := con.CreateTable(ctx, tbl0) + // if assert.Error(t, err) { + // assert.True(t, errors.Is(err, dax.ErrTableIDExists)) + // } exp = []*dax.Directive{} assert.Equal(t, exp, director.flush()) // Add a shard. - assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), - dax.NewVersionedShard(0, 0), - )) + assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), 0)) exp = []*dax.Directive{ { @@ -132,9 +131,7 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(0, 0), - }, + Shards: dax.NewShardNums(0), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -185,13 +182,7 @@ func TestController(t *testing.T) { assert.Equal(t, exp, director.flush()) // Add more shards. - assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), - dax.NewVersionedShard(1, 0), - dax.NewVersionedShard(2, 0), - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(5, 0), - dax.NewVersionedShard(8, 0), - )) + assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), dax.NewShardNums(1, 2, 3, 5, 8)...)) exp = []*dax.Directive{ { @@ -203,10 +194,7 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(0, 0), - dax.NewVersionedShard(3, 0), - }, + Shards: dax.NewShardNums(0, 3), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -221,10 +209,7 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(1, 0), - dax.NewVersionedShard(5, 0), - }, + Shards: dax.NewShardNums(1, 5), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -239,10 +224,7 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(2, 0), - dax.NewVersionedShard(8, 0), - }, + Shards: dax.NewShardNums(2, 8), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -257,12 +239,7 @@ func TestController(t *testing.T) { assert.NoError(t, con.CreateTable(ctx, tbl1)) // Add more shards. - assert.NoError(t, con.AddShards(ctx, tbl1.QualifiedID(), - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(5, 0), - dax.NewVersionedShard(8, 0), - dax.NewVersionedShard(13, 0), - )) + assert.NoError(t, con.AddShards(ctx, tbl1.QualifiedID(), dax.NewShardNums(3, 5, 8, 13)...)) exp = []*dax.Directive{ { @@ -275,17 +252,11 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl1.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(13, 0), - }, + Shards: dax.NewShardNums(3, 13), }, { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(0, 0), - dax.NewVersionedShard(3, 0), - }, + Shards: dax.NewShardNums(0, 3), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -301,16 +272,11 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl1.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(5, 0), - }, + Shards: dax.NewShardNums(5), }, { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(1, 0), - dax.NewVersionedShard(5, 0), - }, + Shards: dax.NewShardNums(1, 5), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -326,16 +292,11 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl1.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(8, 0), - }, + Shards: dax.NewShardNums(8), }, { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(2, 0), - dax.NewVersionedShard(8, 0), - }, + Shards: dax.NewShardNums(2, 8), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -358,18 +319,11 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl1.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(13, 0), - }, + Shards: dax.NewShardNums(3, 13), }, { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(0, 0), - dax.NewVersionedShard(1, 0), - dax.NewVersionedShard(3, 0), - }, + Shards: dax.NewShardNums(0, 1, 3), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -385,18 +339,11 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl1.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(5, 0), - dax.NewVersionedShard(8, 0), - }, + Shards: dax.NewShardNums(5, 8), }, { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(2, 0), - dax.NewVersionedShard(5, 0), - dax.NewVersionedShard(8, 0), - }, + Shards: dax.NewShardNums(2, 5, 8), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -419,23 +366,11 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl1.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(5, 0), - dax.NewVersionedShard(8, 0), - dax.NewVersionedShard(13, 0), - }, + Shards: dax.NewShardNums(3, 5, 8, 13), }, { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(0, 0), - dax.NewVersionedShard(1, 0), - dax.NewVersionedShard(2, 0), - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(5, 0), - dax.NewVersionedShard(8, 0), - }, + Shards: dax.NewShardNums(0, 1, 2, 3, 5, 8), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -470,23 +405,11 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl1.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(5, 0), - dax.NewVersionedShard(8, 0), - dax.NewVersionedShard(13, 0), - }, + Shards: dax.NewShardNums(3, 5, 8, 13), }, { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(0, 0), - dax.NewVersionedShard(1, 0), - dax.NewVersionedShard(2, 0), - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(5, 0), - dax.NewVersionedShard(8, 0), - }, + Shards: dax.NewShardNums(0, 1, 2, 3, 5, 8), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -496,10 +419,7 @@ func TestController(t *testing.T) { assert.Equal(t, exp, director.flush()) // Remove shards. - assert.NoError(t, con.RemoveShards(ctx, tbl0.QualifiedID(), - dax.NewVersionedShard(2, 0), - dax.NewVersionedShard(5, 0), - )) + assert.NoError(t, con.RemoveShards(ctx, tbl0.QualifiedID(), dax.NewShardNums(2, 5)...)) exp = []*dax.Directive{ { @@ -512,21 +432,11 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl1.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(5, 0), - dax.NewVersionedShard(8, 0), - dax.NewVersionedShard(13, 0), - }, + Shards: dax.NewShardNums(3, 5, 8, 13), }, { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(0, 0), - dax.NewVersionedShard(1, 0), - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(8, 0), - }, + Shards: dax.NewShardNums(0, 1, 3, 8), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -538,10 +448,7 @@ func TestController(t *testing.T) { // Remove shards, one which does not exist. // Currently that doesn't result in an error, it simply no-ops on trying // to remove 99. - assert.NoError(t, con.RemoveShards(ctx, tbl0.QualifiedID(), - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(99, 0), - )) + assert.NoError(t, con.RemoveShards(ctx, tbl0.QualifiedID(), dax.NewShardNums(3, 99)...)) exp = []*dax.Directive{ { @@ -554,20 +461,11 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl1.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(5, 0), - dax.NewVersionedShard(8, 0), - dax.NewVersionedShard(13, 0), - }, + Shards: dax.NewShardNums(3, 5, 8, 13), }, { TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(0, 0), - dax.NewVersionedShard(1, 0), - dax.NewVersionedShard(8, 0), - }, + Shards: dax.NewShardNums(0, 1, 8), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -589,12 +487,7 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl1.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(5, 0), - dax.NewVersionedShard(8, 0), - dax.NewVersionedShard(13, 0), - }, + Shards: dax.NewShardNums(3, 5, 8, 13), }, }, TranslateRoles: []dax.TranslateRole{}, @@ -604,7 +497,7 @@ func TestController(t *testing.T) { assert.Equal(t, exp, director.flush()) // Remove a node which doesn't exist. - err = con.DeregisterNodes(ctx, "invalidNode") + err := con.DeregisterNodes(ctx, "invalidNode") if assert.Error(t, err) { assert.True(t, errors.Is(err, dax.ErrNodeDoesNotExist)) } @@ -683,17 +576,8 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(0, 0), - dax.NewVersionedPartition(1, 0), - dax.NewVersionedPartition(2, 0), - dax.NewVersionedPartition(3, 0), - dax.NewVersionedPartition(4, 0), - dax.NewVersionedPartition(5, 0), - dax.NewVersionedPartition(6, 0), - dax.NewVersionedPartition(7, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(0, 1, 2, 3, 4, 5, 6, 7), }, }, Version: 2, @@ -720,13 +604,8 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(0, 0), - dax.NewVersionedPartition(1, 0), - dax.NewVersionedPartition(2, 0), - dax.NewVersionedPartition(3, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(0, 1, 2, 3), }, }, Version: 3, @@ -740,13 +619,8 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(4, 0), - dax.NewVersionedPartition(5, 0), - dax.NewVersionedPartition(6, 0), - dax.NewVersionedPartition(7, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(4, 5, 6, 7), }, }, Version: 4, @@ -772,12 +646,8 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(0, 0), - dax.NewVersionedPartition(1, 0), - dax.NewVersionedPartition(2, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(0, 1, 2), }, }, Version: 5, @@ -791,12 +661,8 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(4, 0), - dax.NewVersionedPartition(5, 0), - dax.NewVersionedPartition(6, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(4, 5, 6), }, }, Version: 6, @@ -810,11 +676,8 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(3, 0), - dax.NewVersionedPartition(7, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(3, 7), }, }, Version: 7, @@ -842,25 +705,12 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl1.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(1, 0), - dax.NewVersionedPartition(4, 0), - dax.NewVersionedPartition(7, 0), - dax.NewVersionedPartition(10, 0), - dax.NewVersionedPartition(13, 0), - dax.NewVersionedPartition(16, 0), - dax.NewVersionedPartition(19, 0), - dax.NewVersionedPartition(22, 0), - }, + TableKey: tbl1.Key(), + Partitions: dax.NewPartitionNums(1, 4, 7, 10, 13, 16, 19, 22), }, { - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(0, 0), - dax.NewVersionedPartition(1, 0), - dax.NewVersionedPartition(2, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(0, 1, 2), }, }, Version: 8, @@ -875,25 +725,12 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl1.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(2, 0), - dax.NewVersionedPartition(5, 0), - dax.NewVersionedPartition(8, 0), - dax.NewVersionedPartition(11, 0), - dax.NewVersionedPartition(14, 0), - dax.NewVersionedPartition(17, 0), - dax.NewVersionedPartition(20, 0), - dax.NewVersionedPartition(23, 0), - }, + TableKey: tbl1.Key(), + Partitions: dax.NewPartitionNums(2, 5, 8, 11, 14, 17, 20, 23), }, { - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(4, 0), - dax.NewVersionedPartition(5, 0), - dax.NewVersionedPartition(6, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(4, 5, 6), }, }, Version: 9, @@ -908,24 +745,12 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl1.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(0, 0), - dax.NewVersionedPartition(3, 0), - dax.NewVersionedPartition(6, 0), - dax.NewVersionedPartition(9, 0), - dax.NewVersionedPartition(12, 0), - dax.NewVersionedPartition(15, 0), - dax.NewVersionedPartition(18, 0), - dax.NewVersionedPartition(21, 0), - }, + TableKey: tbl1.Key(), + Partitions: dax.NewPartitionNums(0, 3, 6, 9, 12, 15, 18, 21), }, { - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(3, 0), - dax.NewVersionedPartition(7, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(3, 7), }, }, Version: 10, @@ -947,17 +772,8 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl1.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(1, 0), - dax.NewVersionedPartition(4, 0), - dax.NewVersionedPartition(7, 0), - dax.NewVersionedPartition(10, 0), - dax.NewVersionedPartition(13, 0), - dax.NewVersionedPartition(16, 0), - dax.NewVersionedPartition(19, 0), - dax.NewVersionedPartition(22, 0), - }, + TableKey: tbl1.Key(), + Partitions: dax.NewPartitionNums(1, 4, 7, 10, 13, 16, 19, 22), }, }, Version: 11, @@ -971,17 +787,8 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl1.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(2, 0), - dax.NewVersionedPartition(5, 0), - dax.NewVersionedPartition(8, 0), - dax.NewVersionedPartition(11, 0), - dax.NewVersionedPartition(14, 0), - dax.NewVersionedPartition(17, 0), - dax.NewVersionedPartition(20, 0), - dax.NewVersionedPartition(23, 0), - }, + TableKey: tbl1.Key(), + Partitions: dax.NewPartitionNums(2, 5, 8, 11, 14, 17, 20, 23), }, }, Version: 12, @@ -995,17 +802,8 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl1.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(0, 0), - dax.NewVersionedPartition(3, 0), - dax.NewVersionedPartition(6, 0), - dax.NewVersionedPartition(9, 0), - dax.NewVersionedPartition(12, 0), - dax.NewVersionedPartition(15, 0), - dax.NewVersionedPartition(18, 0), - dax.NewVersionedPartition(21, 0), - }, + TableKey: tbl1.Key(), + Partitions: dax.NewPartitionNums(0, 3, 6, 9, 12, 15, 18, 21), }, }, Version: 13, @@ -1020,13 +818,11 @@ func TestController(t *testing.T) { } // Add shards to a table which doesn't exist. - err = con.AddShards(ctx, invalidQtid, - dax.NewVersionedShard(1, 0), - dax.NewVersionedShard(2, 0), - ) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist)) - } + // TODO(jaffee) figure out what this should do + // err = con.AddShards(ctx, invalidQtid, dax.NewShardNums(1, 2)...) + // if assert.Error(t, err) { + // assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist)) + // } // Register an invalid node. nodeX := &dax.Node{ @@ -1082,14 +878,7 @@ func TestController(t *testing.T) { assert.NoError(t, con.CreateTable(ctx, tbl0)) // Add shards. - assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), - dax.NewVersionedShard(0, 0), - dax.NewVersionedShard(1, 0), - dax.NewVersionedShard(2, 0), - dax.NewVersionedShard(3, 0), - dax.NewVersionedShard(11, 0), - dax.NewVersionedShard(12, 0), - )) + assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), 0, 1, 2, 3, 11, 12)) t.Run("ComputeRole", func(t *testing.T) { tests := []struct { @@ -1100,27 +889,21 @@ func TestController(t *testing.T) { { role: &dax.ComputeRole{ TableKey: tbl0.Key(), - Shards: dax.NewVersionedShards(0, 1, 2, 3), + Shards: dax.NewShardNums(0, 1, 2, 3), }, exp: []dax.AssignedNode{ { Address: node0.Address, Role: &dax.ComputeRole{ TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(0, 0), - dax.NewVersionedShard(2, 0), - }, + Shards: dax.NewShardNums(0, 2), }, }, { Address: node1.Address, Role: &dax.ComputeRole{ TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(1, 0), - dax.NewVersionedShard(3, 0), - }, + Shards: dax.NewShardNums(1, 3), }, }, }, @@ -1128,16 +911,14 @@ func TestController(t *testing.T) { { role: &dax.ComputeRole{ TableKey: tbl0.Key(), - Shards: dax.NewVersionedShards(1), + Shards: dax.NewShardNums(1), }, exp: []dax.AssignedNode{ { Address: node1.Address, Role: &dax.ComputeRole{ TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(1, 0), - }, + Shards: dax.NewShardNums(1), }, }, }, @@ -1146,7 +927,7 @@ func TestController(t *testing.T) { // Add unassigned shards. role: &dax.ComputeRole{ TableKey: tbl0.Key(), - Shards: dax.NewVersionedShards(1, 888, 889), + Shards: dax.NewShardNums(1, 888, 889), }, isWrite: true, exp: []dax.AssignedNode{ @@ -1154,19 +935,14 @@ func TestController(t *testing.T) { Address: node0.Address, Role: &dax.ComputeRole{ TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(888, 0), - }, + Shards: dax.NewShardNums(888), }, }, { Address: node1.Address, Role: &dax.ComputeRole{ TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(1, 0), - dax.NewVersionedShard(889, 0), - }, + Shards: dax.NewShardNums(1, 889), }, }, }, @@ -1175,17 +951,14 @@ func TestController(t *testing.T) { // Ensure shards are not returned sorted as strings. role: &dax.ComputeRole{ TableKey: tbl0.Key(), - Shards: dax.NewVersionedShards(2, 11), + Shards: dax.NewShardNums(2, 11), }, exp: []dax.AssignedNode{ { Address: node0.Address, Role: &dax.ComputeRole{ TableKey: tbl0.Key(), - Shards: dax.VersionedShards{ - dax.NewVersionedShard(2, 0), - dax.NewVersionedShard(11, 0), - }, + Shards: dax.NewShardNums(2, 11), }, }, }, @@ -1209,75 +982,55 @@ func TestController(t *testing.T) { }{ { role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(0, -1), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(0), }, isWrite: true, exp: []dax.AssignedNode{ { Address: node0.Address, Role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(0, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(0), }, }, }, }, { role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(0, -1), - dax.NewVersionedPartition(1, -1), - dax.NewVersionedPartition(2, -1), - dax.NewVersionedPartition(3, -1), - dax.NewVersionedPartition(999, -1), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(0, 1, 2, 3, 999), }, isWrite: false, exp: []dax.AssignedNode{ { Address: node0.Address, Role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(0, 0), - dax.NewVersionedPartition(2, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(0, 2), }, }, { Address: node1.Address, Role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(1, 0), - dax.NewVersionedPartition(3, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(1, 3), }, }, }, }, { role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(1, -1), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(1), }, isWrite: false, exp: []dax.AssignedNode{ { Address: node1.Address, Role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(1, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(1), }, }, }, @@ -1285,22 +1038,16 @@ func TestController(t *testing.T) { { // Ensure partitions are not returned sorted as strings. role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(2, -1), - dax.NewVersionedPartition(10, -1), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(2, 10), }, isWrite: false, exp: []dax.AssignedNode{ { Address: node0.Address, Role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.VersionedPartitions{ - dax.NewVersionedPartition(2, 0), - dax.NewVersionedPartition(10, 0), - }, + TableKey: tbl0.Key(), + Partitions: dax.NewPartitionNums(2, 10), }, }, }, diff --git a/dax/mds/controller/naive/balancer.go b/dax/mds/controller/naive/balancer.go index 4cc2ec18e..8a8d2a996 100644 --- a/dax/mds/controller/naive/balancer.go +++ b/dax/mds/controller/naive/balancer.go @@ -55,15 +55,18 @@ type WorkerJobService interface { CreateJobs(ctx context.Context, balancerName string, worker dax.Worker, job ...dax.Job) error DeleteJob(ctx context.Context, balancerName string, worker dax.Worker, job dax.Job) error + DeleteJobs(ctx context.Context, balancerName, prefix string) (InternalDiffs, error) JobCounts(ctx context.Context, balancerName string, worker ...dax.Worker) (map[dax.Worker]int, error) ListJobs(ctx context.Context, balancerName string, worker dax.Worker) (dax.Jobs, error) } +// TODO: I don't think all these method names need "Free" in them. type FreeJobService interface { CreateFreeJobs(ctx context.Context, balancerName string, job ...dax.Job) error DeleteFreeJob(ctx context.Context, balancerName string, job dax.Job) error ListFreeJobs(ctx context.Context, balancerName string) (dax.Jobs, error) MergeFreeJobs(ctx context.Context, balancerName string, jobs dax.Jobs) error + DeleteFreeJobs(ctx context.Context, balancerName, prefix string) error } // New returns a new instance of Balancer. @@ -90,15 +93,15 @@ func (b *Balancer) AddWorker(ctx context.Context, worker fmt.Stringer) ([]dax.Wo return nil, errors.Wrap(err, "adding worker") } - return diff.output(), nil + return diff.Output(), nil } -func (b *Balancer) addWorker(ctx context.Context, worker dax.Worker) (internalDiffs, error) { +func (b *Balancer) addWorker(ctx context.Context, worker dax.Worker) (InternalDiffs, error) { // If this worker already exists, don't do anything. if exists, err := b.current.WorkerExists(ctx, b.name, worker); err != nil { return nil, errors.Wrap(err, "checking if worker exists") } else if exists { - return internalDiffs{}, nil + return InternalDiffs{}, nil } if err := b.current.CreateWorker(ctx, b.name, worker); err != nil { @@ -133,15 +136,15 @@ func (b *Balancer) RemoveWorker(ctx context.Context, worker fmt.Stringer) ([]dax return nil, errors.Wrap(err, "removing worker") } - return diff.output(), nil + return diff.Output(), nil } -func (b *Balancer) removeWorker(ctx context.Context, worker dax.Worker) (internalDiffs, error) { +func (b *Balancer) removeWorker(ctx context.Context, worker dax.Worker) (InternalDiffs, error) { // If this worker doesn't exist, don't do anything else. if exists, err := b.current.WorkerExists(ctx, b.name, worker); err != nil { return nil, errors.Wrap(err, "checking if worker exists") } else if !exists { - return internalDiffs{}, nil + return InternalDiffs{}, nil } jobs, err := b.current.ListJobs(ctx, b.name, worker) @@ -162,9 +165,9 @@ func (b *Balancer) removeWorker(ctx context.Context, worker dax.Worker) (interna // Even though this may not be useful to the caller (for example, in the // case where the worker has died and no longer exists), return the diffs // which represent the removal of jobs from the worker. - diff := newInternalDiffs() + diff := NewInternalDiffs() for _, job := range jobs { - diff.removed(worker, job) + diff.Removed(worker, job) } return diff, nil @@ -198,10 +201,10 @@ func (b *Balancer) AddJobs(ctx context.Context, jobs ...fmt.Stringer) ([]dax.Wor return nil, errors.Wrap(err, "adding job") } - return diff.output(), nil + return diff.Output(), nil } -func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (internalDiffs, error) { +func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (InternalDiffs, error) { if cnt, err := b.current.WorkerCount(ctx, b.name); err != nil { return nil, errors.Wrap(err, "getting worker count") } else if cnt == 0 { @@ -210,7 +213,7 @@ func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (internalDiffs, } // TODO: we might want to inform the user that a job is in the free list // because there are no workers. - return internalDiffs{}, nil + return InternalDiffs{}, nil } workerJobs, err := b.current.WorkersJobs(ctx, b.name) @@ -229,7 +232,7 @@ func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (internalDiffs, jobCounts[v.ID] = len(v.Jobs) } - diffs := newInternalDiffs() + diffs := NewInternalDiffs() jobsToCreate := make(map[dax.Worker][]dax.Job) @@ -263,7 +266,7 @@ func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (internalDiffs, return nil, errors.Wrap(err, "creating job") } for _, job := range jobs { - diffs.added(worker, job) + diffs.Added(worker, job) } } @@ -282,10 +285,24 @@ func (b *Balancer) RemoveJob(ctx context.Context, job fmt.Stringer) ([]dax.Worke return nil, errors.Wrapf(err, "removing job: %s", job) } - return diff.output(), nil + return diff.Output(), nil } -func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (internalDiffs, error) { +func (b *Balancer) RemoveJobs(ctx context.Context, prefix string) ([]dax.WorkerDiff, error) { + b.mu.Lock() + defer b.mu.Unlock() + + idiffs, err := b.current.DeleteJobs(ctx, b.name, prefix) + if err != nil { + return nil, errors.Wrap(err, "deleting worker jobs") + } + if err := b.freeJobs.DeleteFreeJobs(ctx, b.name, prefix); err != nil { + return nil, errors.Wrap(err, "deleting free jobs") + } + return idiffs.Output(), nil +} + +func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (InternalDiffs, error) { if worker, ok, err := b.workerForJob(ctx, job); err != nil { return nil, errors.Wrapf(err, "getting worker for job: %s", job) } else if ok { @@ -293,8 +310,8 @@ func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (internalDiffs, e return nil, errors.Wrapf(err, "deleting job: %s", job) } - diffs := newInternalDiffs() - diffs.removed(worker, job) + diffs := NewInternalDiffs() + diffs.Removed(worker, job) return diffs, nil } @@ -307,7 +324,7 @@ func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (internalDiffs, e return nil, errors.Wrapf(err, "deleting free job: %s", job) } - return internalDiffs{}, nil + return InternalDiffs{}, nil } // Balance ensures that all jobs are being handled by a worker by assigning jobs @@ -336,7 +353,7 @@ func (b *Balancer) Balance(ctx context.Context) ([]dax.WorkerDiff, error) { return nil, errors.Wrap(err, "balancing jobs") } - return diff.output(), nil + return diff.Output(), nil } // balance moves jobs among workers with the goal of having an equal number of @@ -346,7 +363,7 @@ func (b *Balancer) Balance(ctx context.Context) ([]dax.WorkerDiff, error) { // the internalDiffs.merge() method, but we would need to modify that method to // be smarter about the order in which it applies the add/remove operations. // Until that's in place, we'll pass in a value here. -func (b *Balancer) balance(ctx context.Context, diffs internalDiffs) (internalDiffs, error) { +func (b *Balancer) balance(ctx context.Context, diffs InternalDiffs) (InternalDiffs, error) { numWorkers, err := b.current.WorkerCount(ctx, b.name) if err != nil { return nil, errors.Wrapf(err, "getting worker count: %s", b.name) @@ -405,12 +422,12 @@ func (b *Balancer) balance(ctx context.Context, diffs internalDiffs) (internalDi if rj, err := b.removeJob(ctx, sortedJobs[i]); err != nil { return nil, errors.Wrapf(err, "removing job: %s", sortedJobs[i]) } else { - diffs.merge(rj) + diffs.Merge(rj) } if aj, err := b.addJobs(ctx, sortedJobs[i]); err != nil { return nil, errors.Wrapf(err, "adding job: %s", sortedJobs[i]) } else { - diffs.merge(aj) + diffs.Merge(aj) } } } @@ -547,8 +564,8 @@ func (b *Balancer) WorkersForJobPrefix(ctx context.Context, prefix string) ([]da } // processFreeJobs assigns all jobs in the free list to a worker. -func (b *Balancer) processFreeJobs(ctx context.Context) (internalDiffs, error) { - diffs := newInternalDiffs() +func (b *Balancer) processFreeJobs(ctx context.Context) (InternalDiffs, error) { + diffs := NewInternalDiffs() jobs, err := b.freeJobs.ListFreeJobs(ctx, b.name) if err != nil { return nil, errors.Wrapf(err, "listing free jobs: %s", b.name) @@ -557,7 +574,7 @@ func (b *Balancer) processFreeJobs(ctx context.Context) (internalDiffs, error) { if aj, err := b.addJobs(ctx, job); err != nil { return nil, errors.Wrapf(err, "adding job: %s", job) } else { - diffs.merge(aj) + diffs.Merge(aj) } if err := b.freeJobs.DeleteFreeJob(ctx, b.name, job); err != nil { return nil, errors.Wrapf(err, "deleting free job: %s", job) diff --git a/dax/mds/controller/naive/boltdb/balancer.go b/dax/mds/controller/naive/boltdb/balancer.go index 93c5ebff7..b8e202913 100644 --- a/dax/mds/controller/naive/boltdb/balancer.go +++ b/dax/mds/controller/naive/boltdb/balancer.go @@ -289,6 +289,53 @@ func (w *workerJobService) DeleteJob(ctx context.Context, balancerName string, w return tx.Commit() } +func (w *workerJobService) DeleteJobs(ctx context.Context, balancerName, prefix string) (naive.InternalDiffs, error) { + tx, err := w.db.BeginTx(ctx, true) + if err != nil { + return nil, errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + bkt := tx.Bucket(bucketNaiveBalancer) + if bkt == nil { + return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) + } + + workers, err := w.getWorkers(ctx, tx, balancerName) + if err != nil { + return nil, errors.Wrap(err, "getting workers") + } + + idiffs := naive.NewInternalDiffs() + for _, worker := range workers { + // get worker + wrkr := bkt.Get(workerKey(balancerName, worker)) + if wrkr == nil { + panic("didn't find worker that should... definitely exist") + } + jobset, err := decodeJobSet(wrkr) + if err != nil { + return nil, errors.Wrap(err, "decoding job set") + } + + jobs := jobset.RemovePrefix(prefix) + for _, job := range jobs { + idiffs.Removed(worker, job) + } + val, err := encodeJobSet(jobset) + if err != nil { + return nil, errors.Wrap(err, "encoding job set") + } + + if err := bkt.Put(workerKey(balancerName, worker), val); err != nil { + return nil, errors.Wrap(err, "putting worker") + } + + } + + return idiffs, tx.Commit() +} + func (w *workerJobService) ListJobs(ctx context.Context, balancerName string, worker dax.Worker) (dax.Jobs, error) { tx, err := w.db.BeginTx(ctx, false) if err != nil { @@ -429,6 +476,42 @@ func (f *freeJobService) DeleteFreeJob(ctx context.Context, balancerName string, return tx.Commit() } +func (f *freeJobService) DeleteFreeJobs(ctx context.Context, balancerName, prefix string) error { + tx, err := f.db.BeginTx(ctx, true) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + bkt := tx.Bucket(bucketNaiveBalancer) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) + } + + // get free jobs + fjs := bkt.Get(freeJobKey(balancerName)) + if fjs == nil { + return nil + } + + jobset, err := decodeJobSet(fjs) + if err != nil { + return errors.Wrap(err, "decoding job set") + } + + jobset.RemovePrefix(prefix) + val, err := encodeJobSet(jobset) + if err != nil { + return errors.Wrap(err, "encoding job set") + } + + if err := bkt.Put(freeJobKey(balancerName), val); err != nil { + return errors.Wrap(err, "putting free job") + } + + return tx.Commit() +} + func (f *freeJobService) ListFreeJobs(ctx context.Context, balancerName string) (dax.Jobs, error) { tx, err := f.db.BeginTx(ctx, false) if err != nil { diff --git a/dax/mds/controller/naive/types.go b/dax/mds/controller/naive/types.go index 6995e24d0..6659b7f9d 100644 --- a/dax/mds/controller/naive/types.go +++ b/dax/mds/controller/naive/types.go @@ -20,13 +20,13 @@ func newJobSetDiffs() jobSetDiffs { } } -type internalDiffs map[dax.Worker]jobSetDiffs +type InternalDiffs map[dax.Worker]jobSetDiffs -func newInternalDiffs() internalDiffs { - return make(internalDiffs) +func NewInternalDiffs() InternalDiffs { + return make(InternalDiffs) } -func (d internalDiffs) added(worker dax.Worker, job dax.Job) { +func (d InternalDiffs) Added(worker dax.Worker, job dax.Job) { if _, ok := d[worker]; !ok { d[worker] = newJobSetDiffs() } @@ -39,7 +39,7 @@ func (d internalDiffs) added(worker dax.Worker, job dax.Job) { d[worker].added.Add(job) } -func (d internalDiffs) removed(worker dax.Worker, job dax.Job) { +func (d InternalDiffs) Removed(worker dax.Worker, job dax.Job) { if _, ok := d[worker]; !ok { d[worker] = newJobSetDiffs() } @@ -52,7 +52,7 @@ func (d internalDiffs) removed(worker dax.Worker, job dax.Job) { d[worker].removed.Add(job) } -func (d internalDiffs) merge(d2 internalDiffs) { +func (d InternalDiffs) Merge(d2 InternalDiffs) { for k, v := range d2 { if _, ok := d[k]; !ok { d[k] = newJobSetDiffs() @@ -62,9 +62,9 @@ func (d internalDiffs) merge(d2 internalDiffs) { } } -// output converts internalDiff to []controller.WorkerDiff for external +// Output converts internalDiff to []controller.WorkerDiff for external // consumption. -func (d internalDiffs) output() []dax.WorkerDiff { +func (d InternalDiffs) Output() []dax.WorkerDiff { out := make([]dax.WorkerDiff, len(d)) i := 0 diff --git a/dax/mds/controller/stringers.go b/dax/mds/controller/stringers.go index fae28e92c..d47150470 100644 --- a/dax/mds/controller/stringers.go +++ b/dax/mds/controller/stringers.go @@ -13,11 +13,11 @@ import ( // used as a job in the Balancer. type pUnit struct { t dax.TableKey - p dax.VersionedPartition + p dax.PartitionNum } func (p pUnit) String() string { - return fmt.Sprintf("%s|part_%d", p.t, p.p.Num) + return fmt.Sprintf("%s|part_%d", p.t, p.p) } func (p pUnit) table() dax.TableKey { @@ -25,14 +25,14 @@ func (p pUnit) table() dax.TableKey { } func (p pUnit) partitionNum() dax.PartitionNum { - return p.p.Num + return p.p } -func partition(t dax.TableKey, p dax.VersionedPartition) pUnit { +func partition(t dax.TableKey, p dax.PartitionNum) pUnit { return pUnit{t, p} } -func partitions(t dax.TableKey, p ...dax.VersionedPartition) []pUnit { +func partitions(t dax.TableKey, p ...dax.PartitionNum) []pUnit { ret := make([]pUnit, 0, len(p)) for _, vp := range p { ret = append(ret, pUnit{t, vp}) @@ -57,10 +57,7 @@ func decodePartition(j dax.Job) (pUnit, error) { return pUnit{ t: dax.TableKey(parts[0]), - p: dax.VersionedPartition{ - Num: dax.PartitionNum(intVar), - Version: -1, - }, + p: dax.PartitionNum(intVar), }, nil } @@ -68,11 +65,11 @@ func decodePartition(j dax.Job) (pUnit, error) { // a job in the Balancer. type sUnit struct { t dax.TableKey - s dax.VersionedShard + s dax.ShardNum } func (s sUnit) String() string { - return fmt.Sprintf("%s|shard_%s", s.t, s.s.Num) + return fmt.Sprintf("%s|shard_%s", s.t, s.s) } func (s sUnit) table() dax.TableKey { @@ -80,10 +77,10 @@ func (s sUnit) table() dax.TableKey { } func (s sUnit) shardNum() dax.ShardNum { - return s.s.Num + return s.s } -func shard(t dax.TableKey, s dax.VersionedShard) sUnit { +func shard(t dax.TableKey, s dax.ShardNum) sUnit { return sUnit{t, s} } @@ -104,9 +101,6 @@ func decodeShard(j dax.Job) (sUnit, error) { return sUnit{ t: dax.TableKey(parts[0]), - s: dax.VersionedShard{ - Num: dax.ShardNum(uint64Var), - Version: -1, - }, + s: dax.ShardNum(uint64Var), }, nil } diff --git a/dax/role.go b/dax/role.go index 48655a873..92712ca64 100644 --- a/dax/role.go +++ b/dax/role.go @@ -33,8 +33,8 @@ var _ Role = &TranslateRole{} // ComputeRole is a role specific to compute nodes. type ComputeRole struct { - TableKey TableKey `json:"table-key"` - Shards VersionedShards `json:"shards"` + TableKey TableKey `json:"table-key"` + Shards ShardNums `json:"shards"` } // Type returns the type for ComputeRole. This is mainly to impolement the Role @@ -45,9 +45,9 @@ func (cr *ComputeRole) Type() RoleType { // TranslateRole is a role specific to translate nodes. type TranslateRole struct { - TableKey TableKey `json:"table-key"` - Partitions VersionedPartitions `json:"partitions"` - Fields VersionedFields `json:"fields"` + TableKey TableKey `json:"table-key"` + Partitions PartitionNums `json:"partitions"` + Fields []FieldName `json:"fields"` } // Type returns the type for TransteRole. This is mainly to impolement the Role diff --git a/dax/versioned_field.go b/dax/versioned_field.go deleted file mode 100644 index 3edfb2608..000000000 --- a/dax/versioned_field.go +++ /dev/null @@ -1,30 +0,0 @@ -package dax - -import "fmt" - -// VersionedField is used in a similar way to VersionedShard and -// VersionedPartition in that they all contain a snapshot version. -type VersionedField struct { - Name FieldName `json:"name"` - Version int `json:"version"` -} - -// String returns the VersionedField (i.e. its Name and Version) as a string. -func (f VersionedField) String() string { - return fmt.Sprintf("%s.%d", f.Name, f.Version) -} - -// NewVersionedField returns a VersionedField with the provided name and version. -func NewVersionedField(name FieldName, version int) VersionedField { - return VersionedField{ - Name: name, - Version: version, - } -} - -// VersionedFields is a sortable slice of VersionedField. -type VersionedFields []VersionedField - -func (f VersionedFields) Len() int { return len(f) } -func (f VersionedFields) Less(i, j int) bool { return f[i].Name < f[j].Name } -func (f VersionedFields) Swap(i, j int) { f[i], f[j] = f[j], f[i] } diff --git a/dax/versioned_partition.go b/dax/versioned_partition.go index 2bc645c8e..32b738893 100644 --- a/dax/versioned_partition.go +++ b/dax/versioned_partition.go @@ -17,55 +17,10 @@ func (p PartitionNum) String() string { return fmt.Sprintf("%d", p) } -// VersionedPartition is a partition number along with the snapshot version -// which it is currently writing at. -type VersionedPartition struct { - Num PartitionNum `json:"num"` - Version int `json:"version"` -} - -// NewVersionedPartition returns a VersionedPartition with the provided -// partition number and version. -func NewVersionedPartition(num PartitionNum, version int) VersionedPartition { - return VersionedPartition{ - Num: num, - Version: version, +func NewPartitionNums(nums ...uint64) PartitionNums { + partitions := make(PartitionNums, len(nums)) + for i, n := range nums { + partitions[i] = PartitionNum(n) } -} - -// String returns the VersionedPartition (i.e. its Num and Version) as a string. -func (p VersionedPartition) String() string { - return fmt.Sprintf("%d.%d", p.Num, p.Version) -} - -// VersionedPartitions is a sortable slice of VersionedPartition. -type VersionedPartitions []VersionedPartition - -func (p VersionedPartitions) Len() int { return len(p) } -func (p VersionedPartitions) Less(i, j int) bool { return p[i].Num < p[j].Num } -func (p VersionedPartitions) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - -// NewVersionedPartitions returns the provided list of partition nums as a list -// of VersionedPartition with an invalid version (-1). This is to use for cases -// where the request should not be aware of a partition versioning. -func NewVersionedPartitions(partitionNums ...PartitionNum) VersionedPartitions { - pvs := make(VersionedPartitions, len(partitionNums)) - - for i := range partitionNums { - pvs[i] = VersionedPartition{ - Num: partitionNums[i], - Version: -1, - } - } - - return pvs -} - -// Nums returns a slice of all the partition numbers in VersionedPartitions. -func (p VersionedPartitions) Nums() []PartitionNum { - pp := make([]PartitionNum, len(p)) - for i := range p { - pp[i] = p[i].Num - } - return pp + return partitions } diff --git a/dax/versioned_shard.go b/dax/versioned_shard.go index 68d568179..865b55022 100644 --- a/dax/versioned_shard.go +++ b/dax/versioned_shard.go @@ -12,54 +12,14 @@ func (s ShardNum) String() string { return fmt.Sprintf("%d", s) } -// VersionedShard is a shard number along with the snapshot version which it is -// currently writing at. -type VersionedShard struct { - Num ShardNum `json:"num"` - Version int `json:"version"` -} +func (s ShardNums) Len() int { return len(s) } +func (s ShardNums) Less(i, j int) bool { return s[i] < s[j] } +func (s ShardNums) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -// NewVersionedShard returns a VersionedShard with the provided shard number and version. -func NewVersionedShard(num ShardNum, version int) VersionedShard { - return VersionedShard{ - Num: num, - Version: version, +func NewShardNums(nums ...uint64) ShardNums { + shards := make(ShardNums, len(nums)) + for i, n := range nums { + shards[i] = ShardNum(n) } -} - -// String returns the VersionedShard (i.e. its Num and Version) as a string. -func (s VersionedShard) String() string { - return fmt.Sprintf("%d.%d", s.Num, s.Version) -} - -// VersionedShards is a sortable slice of VersionedShard. -type VersionedShards []VersionedShard - -func (s VersionedShards) Len() int { return len(s) } -func (s VersionedShards) Less(i, j int) bool { return s[i].Num < s[j].Num } -func (s VersionedShards) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - -// NewVersionedShards returns the provided list of shard nums as a list of -// VersionedShard with an invalid version (-1). This is to use for cases where -// the request should not be aware of shard versioning. -func NewVersionedShards(shardNums ...ShardNum) VersionedShards { - svs := make(VersionedShards, len(shardNums)) - - for i := range shardNums { - svs[i] = VersionedShard{ - Num: shardNums[i], - Version: -1, - } - } - - return svs -} - -// Nums returns a slice of all the shard numbers in VersionedShards. -func (s VersionedShards) Nums() []ShardNum { - ss := make([]ShardNum, len(s)) - for i := range s { - ss[i] = s[i].Num - } - return ss + return shards } diff --git a/dax/versionstore.go b/dax/versionstore.go deleted file mode 100644 index 1d1dc051f..000000000 --- a/dax/versionstore.go +++ /dev/null @@ -1,110 +0,0 @@ -package dax - -import ( - "context" -) - -// VersionStore is an interface for tracking Shard, Partition, and Field[Key] -// versions. For example, when the contents of a shard are checkpointed, and a -// snapshot is generated, and the write log messages for that shard are -// truncated, the ShardVersion for that shard is incremented. The VersionStore -// is the interface through which various services read/write that version. -type VersionStore interface { - AddTable(ctx context.Context, qtid QualifiedTableID) error - RemoveTable(ctx context.Context, qtid QualifiedTableID) (VersionedShards, VersionedPartitions, error) - - // Shards (shardData) - AddShards(ctx context.Context, qtid QualifiedTableID, shards ...VersionedShard) error - Shards(ctx context.Context, qtid QualifiedTableID) (VersionedShards, bool, error) - ShardVersion(ctx context.Context, qtid QualifiedTableID, shardNum ShardNum) (int, bool, error) - ShardTables(ctx context.Context, qual TableQualifier) (TableIDs, error) - - // Partitions (tableKeys) - AddPartitions(ctx context.Context, qtid QualifiedTableID, partitions ...VersionedPartition) error - Partitions(ctx context.Context, qtid QualifiedTableID) (VersionedPartitions, bool, error) - PartitionVersion(ctx context.Context, qtid QualifiedTableID, partitionNum PartitionNum) (int, bool, error) - PartitionTables(ctx context.Context, qual TableQualifier) (TableIDs, error) - - // Fields (fieldKeys) - AddFields(ctx context.Context, qtid QualifiedTableID, fields ...VersionedField) error - Fields(ctx context.Context, qtid QualifiedTableID) (VersionedFields, bool, error) - FieldVersion(ctx context.Context, qtid QualifiedTableID, field FieldName) (int, bool, error) - FieldTables(ctx context.Context, qual TableQualifier) (TableIDs, error) - - Copy(ctx context.Context) (VersionStore, error) -} - -type DirectiveVersion interface { - Increment(ctx context.Context, delta uint64) (uint64, error) -} - -// Ensure type implements interface. -var _ VersionStore = (*nopVersionStore)(nil) - -// nopVersionStore is a no-op implementation of the VersionStore interface. -type nopVersionStore struct{} - -// NewNopVersionStore returns a new no-op instance of VersionStore. -func NewNopVersionStore() *nopVersionStore { - return &nopVersionStore{} -} - -func (s *nopVersionStore) AddTable(ctx context.Context, qtid QualifiedTableID) error { - return nil -} - -func (s *nopVersionStore) RemoveTable(ctx context.Context, qtid QualifiedTableID) (VersionedShards, VersionedPartitions, error) { - return nil, nil, nil -} - -func (s *nopVersionStore) AddShards(ctx context.Context, qtid QualifiedTableID, shards ...VersionedShard) error { - return nil -} - -func (s *nopVersionStore) Shards(ctx context.Context, qtid QualifiedTableID) (VersionedShards, bool, error) { - return nil, false, nil -} - -func (s *nopVersionStore) ShardVersion(ctx context.Context, qtid QualifiedTableID, shardNum ShardNum) (int, bool, error) { - return 0, true, nil -} - -func (s *nopVersionStore) ShardTables(ctx context.Context, qual TableQualifier) (TableIDs, error) { - return TableIDs{}, nil -} - -func (s *nopVersionStore) AddPartitions(ctx context.Context, qtid QualifiedTableID, partitions ...VersionedPartition) error { - return nil -} - -func (s *nopVersionStore) Partitions(ctx context.Context, qtid QualifiedTableID) (VersionedPartitions, bool, error) { - return nil, false, nil -} - -func (s *nopVersionStore) PartitionVersion(ctx context.Context, qtid QualifiedTableID, partitionNum PartitionNum) (int, bool, error) { - return 0, true, nil -} - -func (s *nopVersionStore) PartitionTables(ctx context.Context, qual TableQualifier) (TableIDs, error) { - return TableIDs{}, nil -} - -func (s *nopVersionStore) AddFields(ctx context.Context, qtid QualifiedTableID, fields ...VersionedField) error { - return nil -} - -func (s *nopVersionStore) Fields(ctx context.Context, qtid QualifiedTableID) (VersionedFields, bool, error) { - return nil, false, nil -} - -func (s *nopVersionStore) FieldVersion(ctx context.Context, qtid QualifiedTableID, field FieldName) (int, bool, error) { - return 0, true, nil -} - -func (s *nopVersionStore) FieldTables(ctx context.Context, qual TableQualifier) (TableIDs, error) { - return TableIDs{}, nil -} - -func (s *nopVersionStore) Copy(ctx context.Context) (VersionStore, error) { - return nil, nil -} diff --git a/dax/workerjob.go b/dax/workerjob.go index 5e9c6b93f..a249bd82b 100644 --- a/dax/workerjob.go +++ b/dax/workerjob.go @@ -2,8 +2,7 @@ package dax import ( "sort" - - "golang.org/x/exp/constraints" + "strings" ) // Worker is a generic identifier used to represent a service responsible for @@ -78,10 +77,10 @@ func (w WorkerDiffs) Len() int { return len(w) } func (w WorkerDiffs) Less(i, j int) bool { return w[i].WorkerID < w[j].WorkerID } func (w WorkerDiffs) Swap(i, j int) { w[i], w[j] = w[j], w[i] } -// Set is a set of orderable items. -type Set[K constraints.Ordered] map[K]struct{} +// Set is a set of stringy items. +type Set[K ~string] map[K]struct{} -func NewSet[K constraints.Ordered](stuff ...K) Set[K] { +func NewSet[K ~string](stuff ...K) Set[K] { s := make(map[K]struct{}) for _, thing := range stuff { s[thing] = struct{}{} @@ -110,6 +109,17 @@ func (s Set[K]) Remove(k K) { delete(s, k) } +func (s Set[K]) RemovePrefix(prefix string) []K { + ret := make([]K, 0) + for k := range s { + if strings.HasPrefix(string(k), prefix) { + ret = append(ret, k) + delete(s, k) + } + } + return ret +} + // Slice returns a slice containing each member of the set in an undefined order. func (s Set[K]) Slice() []K { ret := make([]K, 0, len(s)) diff --git a/holder.go b/holder.go index f6004f348..89533d2fc 100644 --- a/holder.go +++ b/holder.go @@ -147,8 +147,6 @@ type Holder struct { // snapshotter/writelogger; then MDS should only start directing queries to // that computer once it has completed applying the snapshot. directiveApplied bool - - versionStore dax.VersionStore } // HolderOpts holds information about the holder which other things might want @@ -344,8 +342,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { path: path, indexes: make(map[string]*Index), - - versionStore: dax.NewNopVersionStore(), } txf, err := NewTxFactory(cfg.StorageConfig.Backend, h.IndexesPath(), h) @@ -1018,14 +1014,6 @@ func (h *Holder) createIndex(cim *CreateIndexMessage, broadcast bool) (*Index, e // Update options. h.addIndex(index) - tkey := dax.TableKey(cim.Index) - qtid := tkey.QualifiedTableID() - - // Initialize the table in holder.versionStore. - if err := h.versionStore.AddTable(context.Background(), qtid); err != nil { - h.Logger.Printf("could not add table to version store: %s", cim.Index) - } - if broadcast { // Send the create index message to all nodes. if err := h.broadcaster.SendSync(cim); err != nil { @@ -1045,7 +1033,7 @@ func (h *Holder) createIndex(cim *CreateIndexMessage, broadcast bool) (*Index, e // createIndexWithPartitions is similar to createIndex, but it takes a list of // partitions for which this node is responsible. This ensures that the node // doesn't instantiate more partition TranslateStores than is necessary. -func (h *Holder) createIndexWithPartitions(cim *CreateIndexMessage, translatePartitions dax.VersionedPartitions) (*Index, error) { +func (h *Holder) createIndexWithPartitions(cim *CreateIndexMessage, translatePartitions dax.PartitionNums) (*Index, error) { if cim.Index == "" { return nil, errors.New("index name required") } @@ -1068,24 +1056,6 @@ func (h *Holder) createIndexWithPartitions(cim *CreateIndexMessage, translatePar // Update options. h.addIndex(index) - tkey := dax.TableKey(cim.Index) - qtid := tkey.QualifiedTableID() - - // Initialize the table in holder.versionStore. - if err := h.versionStore.AddTable(context.Background(), qtid); err != nil { - h.Logger.Printf("could not add table to version store: %s", cim.Index) - } - - // Initialize a list of partitions at version 0. - newPartitions := make(dax.VersionedPartitions, len(translatePartitions)) - for i := range translatePartitions { - newPartitions[i] = dax.NewVersionedPartition(translatePartitions[i].Num, 0) - } - - if err := h.versionStore.AddPartitions(context.Background(), qtid, newPartitions...); err != nil { - return nil, errors.Wrap(err, "adding partitions to version store") - } - // Since this is a new index, we need to kick off // its translation sync. if err := h.translationSyncer.Reset(); err != nil { @@ -1242,14 +1212,6 @@ func (h *Holder) deleteIndex(name string) error { // Remove reference. h.deleteIndexFromMap(name) - tkey := dax.TableKey(name) - qtid := tkey.QualifiedTableID() - - // Remove the index from holder.versionStore. - if _, _, err := h.versionStore.RemoveTable(context.Background(), qtid); err != nil { - h.Logger.Printf("could not find table to remove from version store: %s", name) - } - // I'm not sure if calling Reset() here is necessary // since closing the index stops its translation // sync processes. diff --git a/index.go b/index.go index b49bc1036..33078bca0 100644 --- a/index.go +++ b/index.go @@ -50,7 +50,7 @@ type Index struct { holder *Holder // Per-partition translation stores - translatePartitions dax.VersionedPartitions + translatePartitions dax.PartitionNums translateStores map[int]TranslateStore translationSyncer TranslationSyncer @@ -977,7 +977,7 @@ func (i *Index) DeleteField(name string) error { // case, we need to update this cached value. Really, this is kind of hacky and // we need to revisit the ApplyDirective logic so that it's more intuitive with // respect to index.translatePartitions. -func (i *Index) SetTranslatePartitions(tp dax.VersionedPartitions) { +func (i *Index) SetTranslatePartitions(tp dax.PartitionNums) { i.mu.Lock() defer i.mu.Unlock() diff --git a/server.go b/server.go index eb253add3..9ffc9ccd6 100644 --- a/server.go +++ b/server.go @@ -551,15 +551,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.holder.Logger.Infof("cwd: %v", cwd) s.holder.Logger.Infof("cmd line: %v", strings.Join(os.Args, " ")) - // The compute nodes keep a local cache of the VersionStore which applies - // only to the data (shard, partitions, fields) managed by the compute node - // (as opposed to the VersionStore in MDS which keeps information about all - // data). It would be okay for this to be an in-memory implementation as - // long as the compute node isn't expected to survive a restart; in that - // case, it would be necessary to use an implementation which saves state - // somewhere, such as local disk. - versionStore := inmem.NewVersionStore() - s.cluster.Path = path s.cluster.logger = s.logger s.cluster.holder = s.holder @@ -567,7 +558,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.cluster.noder = s.noder s.cluster.sharder = s.sharder s.cluster.serverlessStorage = s.serverlessStorage - s.cluster.versionStore = versionStore // Append the NodeID tag to stats. s.holder.Stats = s.holder.Stats.WithTags(fmt.Sprintf("node_id:%s", s.nodeID)) @@ -583,7 +573,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.holder.broadcaster = s s.holder.sharder = s.sharder s.holder.serializer = s.serializer - s.holder.versionStore = versionStore // Initial stats must be invoked after the executor obtains reference to the holder. s.executor.InitStats()