scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-06 19:07:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #2204 from FeatureBaseDB/1-10-2023-cherry-pick

Browse source 
v3.27.0 cherry pick from private
This commit is contained in:
Joseph Friedrich 2023-01-12 16:05:13 -06:00 • committed by GitHub
commit 5274caece8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
202 changed files with 10068 additions and 7509 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
LICENSE Normal file
View file

@ -0,0 +1,13 @@
Copyright 2023 Molecula Corp. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

18
Makefile
View file

@ -1,5 +1,6 @@
.PHONY: build clean build-lattice cover cover-viz default docker docker-build docker-tag-push generate generate-protoc generate-pql generate-statik generate-stringer install install-protoc-gen-gofast install-protoc install-statik install-peg test docker-login
SHELL := /bin/bash
VERSION := $(shell git describe --tags 2> /dev/null || echo unknown)
VARIANT = Molecula
GO=go
@ -76,6 +77,16 @@ testvsub:
echo; echo "999 done testing subpkg $$pkg"; \
done
# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/mnt/ramdisk
ramdisk-linux:
mount -o size=2G -t tmpfs none /mnt/ramdisk
# make a 2GB RAMDisk. Speed up tests by running them with RAMDISK=/Volumes/RAMDisk
ramdisk-osx:
diskutil erasevolume HFS+ 'RAMDisk' `hdiutil attach -nobrowse -nomount ram://4194304`
detach-ramdisk-osx:
hdiutil detach /Volumes/RAMDisk
testvsub-race:
@set -e; for pkg in $(GOPACKAGES); do \
@ -228,15 +239,16 @@ build-for-quick:
docker-image-featurebase-quick: build-for-quick
docker build \
--build-arg GO_VERSION=$(GO_VERSION) \
--file Dockerfile-dax-quick ./.quick/
--file Dockerfile-dax-quick \
--tag dax/featurebase ./.quick/
docker-image-datagen: vendor
docker build --tag dax/datagen --file Dockerfile-datagen .
ecr-push-featurebase: docker-login
docker tag dax/featurebase:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax:latest
docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax:latest
docker tag dax/featurebase:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
docker push $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/featurebase:latest
ecr-push-datagen: docker-login
docker tag dax/datagen:latest $(AWS_ACCOUNTID).dkr.ecr.us-east-2.amazonaws.com/dax/datagen:latest

314
api.go
View file

@ -25,6 +25,8 @@ import (
fbcontext "github.com/featurebasedb/featurebase/v3/context"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/computer"
"github.com/featurebasedb/featurebase/v3/dax/storage"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/rbf"
@ -55,9 +57,7 @@ type API struct {
Serializer Serializer
writeLogReader computer.WriteLogReader
writeLogWriter computer.WriteLogWriter
snapshotReadWriter computer.SnapshotReadWriter
serverlessStorage *storage.ResourceManager
directiveWorkerPoolSize int
@ -70,6 +70,10 @@ func (api *API) Holder() *Holder {
return api.holder
}
func (api *API) logger() logger.Logger {
return api.server.logger
}
// apiOption is a functional option type for pilosa.API
type apiOption func(*API) error
@ -83,6 +87,13 @@ func OptAPIServer(s *Server) apiOption {
}
}
func OptAPIServerlessStorage(mm *storage.ResourceManager) apiOption {
return func(a *API) error {
a.serverlessStorage = mm
return nil
}
}
func OptAPIImportWorkerPoolSize(size int) apiOption {
return func(a *API) error {
a.importWorkerPoolSize = size
@ -90,27 +101,6 @@ func OptAPIImportWorkerPoolSize(size int) apiOption {
}
}
func OptAPIWriteLogReader(wlr computer.WriteLogReader) apiOption {
return func(a *API) error {
a.writeLogReader = wlr
return nil
}
}
func OptAPIWriteLogWriter(wlw computer.WriteLogWriter) apiOption {
return func(a *API) error {
a.writeLogWriter = wlw
return nil
}
}
func OptAPISnapshotter(snap computer.SnapshotReadWriter) apiOption {
return func(a *API) error {
a.snapshotReadWriter = snap
return nil
}
}
func OptAPIDirectiveWorkerPoolSize(size int) apiOption {
return func(a *API) error {
a.directiveWorkerPoolSize = size
@ -129,9 +119,6 @@ func OptAPIIsComputeNode(is bool) apiOption {
func NewAPI(opts ...apiOption) (*API, error) {
api := &API{
importWorkerPoolSize: 2,
writeLogReader: computer.NewNopWriteLogReader(),
writeLogWriter: computer.NewNopWriteLogWriter(),
snapshotReadWriter: computer.NewNopSnapshotReadWriter(),
directiveWorkerPoolSize: 2,
}
@ -250,7 +237,7 @@ func (api *API) query(ctx context.Context, req *QueryRequest) (QueryResponse, er
EmbeddedData: req.EmbeddedData, // precomputed values that needed to be passed with the request
MaxMemory: req.MaxMemory,
}
resp, err := api.server.executor.Execute(ctx, req.Index, q, req.Shards, execOpts)
resp, err := api.server.executor.Execute(ctx, dax.StringTableKeyer(req.Index), q, req.Shards, execOpts)
if err != nil {
return QueryResponse{}, errors.Wrap(err, "executing")
}
@ -709,20 +696,20 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
Views: req.Views,
}
// Get the current version for shard.
version, err := api.getOrCreateShardVersion(ctx, indexName, shard)
if err != nil {
return errors.Wrap(err, "get or creating shard version")
}
tkey := dax.TableKey(indexName)
qtid := tkey.QualifiedTableID()
partitionNum := dax.PartitionNum(partition)
shardNum := dax.ShardNum(shard)
api.server.logger.Debugf("importroaring writing to writelogger: %+v, %[1]T len(msg.Views): %d, table: %s", api.writeLogWriter, len(msg.Views), msg.Table)
if err := api.writeLogWriter.WriteShard(ctx, qtid, partitionNum, shardNum, version, msg); err != nil {
return err
b, err := computer.MarshalLogMessage(msg, computer.EncodeTypeJSON)
if err != nil {
return errors.Wrap(err, "marshalling log message")
}
resource := api.serverlessStorage.GetShardResource(qtid, partitionNum, shardNum)
err = resource.Append(b)
if err != nil {
return errors.Wrap(err, "appending shard data") // TODO do we need to set err0 or something?
}
}
@ -731,29 +718,6 @@ func (api *API) ImportRoaring(ctx context.Context, indexName, fieldName string,
}
}
func (api *API) getOrCreateShardVersion(ctx context.Context, indexName string, shard uint64) (int, error) {
tableName := dax.TableName(indexName)
shardNum := dax.ShardNum(shard)
// Here we assume that indexName is the string encoding of QualifiedTableID.
qtid, err := dax.QualifiedTableIDFromKey(indexName)
if err != nil {
return -1, errors.Wrap(err, "decoding qtid from key (indexName)")
}
version, found, err := api.holder.versionStore.ShardVersion(ctx, qtid, shardNum)
if err != nil {
return -1, errors.Wrap(err, "getting shard version")
} else if !found {
version = 0
api.server.logger.Printf("could not find version for shard: %s, %d, so creating 0", tableName, shardNum)
if err := api.holder.versionStore.AddShards(ctx, qtid, dax.NewVersionedShard(shardNum, version)); err != nil {
return -1, errors.Wrap(err, "adding shard 0")
}
}
return version, nil
}
// DeleteField removes the named field from the named index. If the index is not
// found, an error is returned. If the field is not found, it is ignored and no
// action is taken.
@ -982,7 +946,7 @@ func (r RedirectError) Error() string {
}
// TranslateData returns all translation data in the specified partition.
func (api *API) TranslateData(ctx context.Context, indexName string, partition int) (io.WriterTo, error) {
func (api *API) TranslateData(ctx context.Context, indexName string, partition int) (TranslateStore, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.TranslateData")
defer span.Finish()
@ -1037,7 +1001,7 @@ func (api *API) TranslateData(ctx context.Context, indexName string, partition i
}
// FieldTranslateData returns all translation data in the specified field.
func (api *API) FieldTranslateData(ctx context.Context, indexName, fieldName string) (io.WriterTo, error) {
func (api *API) FieldTranslateData(ctx context.Context, indexName, fieldName string) (TranslateStore, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.FieldTranslateData")
defer span.Finish()
if err := api.validate(apiFieldTranslateData); err != nil {
@ -1286,8 +1250,13 @@ func (api *API) DeleteView(ctx context.Context, indexName string, fieldName stri
return errors.Wrap(err, "sending DeleteView message")
}
// IndexShardSnapshot returns a reader that contains the contents of an RBF snapshot for an index/shard.
func (api *API) IndexShardSnapshot(ctx context.Context, indexName string, shard uint64) (io.ReadCloser, error) {
// IndexShardSnapshot returns a reader that contains the contents of
// an RBF snapshot for an index/shard. When snapshotting for
// serverless, we need to be able to transactionally move the write
// log to the new version, so we expose writeTx to allow the caller to
// request a write transaction for the snapshot even though we'll just
// be reading inside RBF.
func (api *API) IndexShardSnapshot(ctx context.Context, indexName string, shard uint64, writeTx bool) (io.ReadCloser, error) {
span, _ := tracing.StartSpanFromContext(ctx, "API.IndexShardSnapshot")
defer span.Finish()
@ -1298,7 +1267,7 @@ func (api *API) IndexShardSnapshot(ctx context.Context, indexName string, shard
}
// Start transaction.
tx := index.holder.txf.NewTx(Txo{Index: index, Shard: shard})
tx := index.holder.txf.NewTx(Txo{Index: index, Shard: shard, Write: writeTx})
// Ensure transaction is an RBF transaction.
rtx, ok := tx.(*RBFTx)
@ -1517,20 +1486,20 @@ func (api *API) Import(ctx context.Context, qcx *Qcx, req *ImportRequest, opts .
}
if api.isComputeNode && !options.suppressLog {
// Get the current version for shard.
version, err := api.getOrCreateShardVersion(ctx, req.Index, req.Shard)
if err != nil {
return errors.Wrap(err, "get or creating shard version")
}
tkey := dax.TableKey(req.Index)
qtid := tkey.QualifiedTableID()
partitionNum := dax.PartitionNum(partition)
shardNum := dax.ShardNum(req.Shard)
// Write the request to the write logger.
if err := api.writeLogWriter.WriteShard(ctx, qtid, partitionNum, shardNum, version, msg); err != nil {
return err
b, err := computer.MarshalLogMessage(msg, computer.EncodeTypeJSON)
if err != nil {
return errors.Wrap(err, "marshalling log message")
}
resource := api.serverlessStorage.GetShardResource(qtid, partitionNum, shardNum)
err = resource.Append(b)
if err != nil {
return errors.Wrap(err, "appending shard data") // TODO do we need to set err0 or something?
}
}
@ -1765,21 +1734,21 @@ func (api *API) ImportRoaringShard(ctx context.Context, indexName string, shard
ClearRecords: view.ClearRecords,
}
}
// Get the current version for shard.
version, err := api.getOrCreateShardVersion(ctx, indexName, shard)
if err != nil {
err1 = errors.Wrap(err, "get or creating shard version")
return err1
}
tkey := dax.TableKey(indexName)
qtid := tkey.QualifiedTableID()
partitionNum := dax.PartitionNum(partition)
shardNum := dax.ShardNum(shard)
api.server.logger.Debugf("importroaringshard writing shard to writelogger: %+v, len(msg.Views): %d, table: %s", api.writeLogWriter, len(msg.Views), msg.Table)
b, err := computer.MarshalLogMessage(msg, computer.EncodeTypeJSON)
if err != nil {
err1 = errors.Wrap(err, "marshalling log message")
return err1
}
if err := api.writeLogWriter.WriteShard(ctx, qtid, partitionNum, shardNum, version, msg); err != nil {
err1 = errors.Wrap(err, "writing import-roaring-shard to writelogger")
resource := api.serverlessStorage.GetShardResource(qtid, partitionNum, shardNum)
err1 = errors.Wrap(resource.Append(b), "appending shard data")
if err1 != nil {
return err1
}
}
@ -1860,20 +1829,21 @@ func (api *API) ImportValue(ctx context.Context, qcx *Qcx, req *ImportValueReque
if api.isComputeNode && !options.suppressLog {
// Get the current version for shard.
version, err := api.getOrCreateShardVersion(ctx, req.Index, req.Shard)
if err != nil {
return errors.Wrap(err, "get or creating shard version")
}
tkey := dax.TableKey(req.Index)
qtid := tkey.QualifiedTableID()
partitionNum := dax.PartitionNum(partition)
shardNum := dax.ShardNum(req.Shard)
// Write the request to the write logger.
if err := api.writeLogWriter.WriteShard(ctx, qtid, partitionNum, shardNum, version, msg); err != nil {
return errors.Wrap(err, "writing shard to write logger")
b, err := computer.MarshalLogMessage(msg, computer.EncodeTypeJSON)
if err != nil {
return errors.Wrap(err, "marshalling log message")
}
resource := api.serverlessStorage.GetShardResource(qtid, partitionNum, shardNum)
err = resource.Append(b)
if err != nil {
return errors.Wrap(err, "appending shard data") // TODO do we need to set err0 or something?
}
}
return nil
@ -3107,52 +3077,43 @@ func (api *API) DirectiveApplied(ctx context.Context) (bool, error) {
// SnapshotShardData triggers the node to perform a shard snapshot based on the
// provided SnapshotShardDataRequest.
func (api *API) SnapshotShardData(ctx context.Context, req *dax.SnapshotShardDataRequest) error {
qtid := req.TableKey.QualifiedTableID()
// Confirm that this node is currently responsible for table/shard/fromVersion.
var version int
if v, ok, err := api.holder.versionStore.ShardVersion(ctx, qtid, req.ShardNum); err != nil {
return err
} else if !ok {
return errors.Errorf("shard not managed by this node: %s, %d", req.TableKey, req.ShardNum)
} else if v != req.FromVersion {
return errors.Errorf("shard managed by this node is at version: %d, not: %d", v, req.FromVersion)
} else {
version = v
if !api.holder.DirectiveApplied() {
return errors.New("don't have directive yet, can't snapshot shard")
}
// TODO(jaffee) confirm this node is actually responsible for the given
// shard? Not sure we need to given that this request comes from
// MDS, but might be a belt&suspenders situation.
qtid := req.TableKey.QualifiedTableID()
partition := disco.ShardToShardPartition(string(req.TableKey), uint64(req.ShardNum), disco.DefaultPartitionN)
partitionNum := dax.PartitionNum(partition)
// Create the snapshot for the current version.
rc, err := api.IndexShardSnapshot(ctx, string(req.TableKey), uint64(req.ShardNum))
// Open a write Tx snapshotting current version.
rc, err := api.IndexShardSnapshot(ctx, string(req.TableKey), uint64(req.ShardNum), true)
if err != nil {
return errors.Wrap(err, "getting index/shard readcloser")
}
defer rc.Close()
// The following closes rc, the ReadCloser.
if err := api.snapshotReadWriter.WriteShardData(ctx, qtid, partitionNum, req.ShardNum, version, rc); err != nil {
return errors.Wrap(err, "snapshotting shard data")
resource := api.serverlessStorage.GetShardResource(qtid, partitionNum, req.ShardNum)
// Bump writelog version while write Tx is held.
if ok, err := resource.IncrementWLVersion(); err != nil {
return errors.Wrap(err, "incrementing write log version")
} else if !ok {
return nil
}
// Increment the version of the shard managed by this node.
if err := api.holder.versionStore.AddShards(ctx, qtid,
dax.NewVersionedShard(req.ShardNum, req.ToVersion),
); err != nil {
return errors.Wrap(err, "incrementing shard version locally")
}
// Update the cached directive on the holder.
api.holder.SetDirective(&req.Directive)
api.holder.SetDirectiveApplied(true)
// Finally, delete the log file for the previous version.
return api.writeLogWriter.DeleteShard(ctx, qtid, partitionNum, req.ShardNum, req.FromVersion)
// TODO(jaffee) look into downgrading Tx on RBF to read lock here now that WL version is incremented.
err = resource.Snapshot(rc)
return errors.Wrap(err, "snapshotting shard data")
}
// SnapshotTableKeys triggers the node to perform a table keys snapshot based on
// the provided SnapshotTableKeysRequest.
func (api *API) SnapshotTableKeys(ctx context.Context, req *dax.SnapshotTableKeysRequest) error {
if !api.holder.DirectiveApplied() {
return errors.New("don't have directive yet, can't snapshot table keys")
}
// If the index is not keyed, no-op on snapshotting its keys.
if idx, err := api.Index(ctx, string(req.TableKey)); err != nil {
return newNotFoundError(ErrIndexNotFound, string(req.TableKey))
@ -3162,83 +3123,60 @@ func (api *API) SnapshotTableKeys(ctx context.Context, req *dax.SnapshotTableKey
qtid := req.TableKey.QualifiedTableID()
// Confirm that this node is currently responsible for table/partition/fromVersion.
var version int
if v, ok, err := api.holder.versionStore.PartitionVersion(ctx, qtid, req.PartitionNum); err != nil {
return err
} else if !ok {
return errors.Errorf("partition not managed by this node: %s, %d", req.TableKey, req.PartitionNum)
} else if v != req.FromVersion {
return errors.Errorf("partition managed by this node is at version: %d, not: %d", v, req.FromVersion)
} else {
version = v
}
// Create the snapshot for the current version.
wrTo, err := api.TranslateData(ctx, string(req.TableKey), int(req.PartitionNum))
trans, err := api.TranslateData(ctx, string(req.TableKey), int(req.PartitionNum))
if err != nil {
return errors.Wrapf(err, "getting index/partition writeto: %s/%d", req.TableKey, req.PartitionNum)
return errors.Wrapf(err, "getting index/partition translate store: %s/%d", req.TableKey, req.PartitionNum)
}
if err := api.snapshotReadWriter.WriteTableKeys(ctx, qtid, req.PartitionNum, version, wrTo); err != nil {
return errors.Wrap(err, "snapshotting table keys")
// get a write tx to ensure no other writes while incrementing WL version.
wrTo, err := trans.Begin(true)
if err != nil {
return errors.Wrap(err, "beginning table translate write tx")
}
defer wrTo.Rollback()
// Increment the version of the partition managed by this node.
if err := api.holder.versionStore.AddPartitions(ctx, qtid,
dax.NewVersionedPartition(req.PartitionNum, req.ToVersion),
); err != nil {
return errors.Wrap(err, "incrementing partition version locally")
resource := api.serverlessStorage.GetTableKeyResource(qtid, req.PartitionNum)
if ok, err := resource.IncrementWLVersion(); err != nil {
return errors.Wrap(err, "incrementing write log version")
} else if !ok {
// no need to snapshot, no writes
return nil
}
// Update the cached directive on the holder.
api.holder.SetDirective(&req.Directive)
api.holder.SetDirectiveApplied(true)
// Finally, delete the log file for the previous version.
return api.writeLogWriter.DeleteTableKeys(ctx, qtid, req.PartitionNum, req.FromVersion)
// TODO(jaffee) downgrade write tx to read-only
err = resource.SnapshotTo(wrTo)
return errors.Wrap(err, "snapshotting table keys")
}
// SnapshotFieldKeys triggers the node to perform a field keys snapshot based on
// the provided SnapshotFieldKeysRequest.
func (api *API) SnapshotFieldKeys(ctx context.Context, req *dax.SnapshotFieldKeysRequest) error {
if !api.holder.DirectiveApplied() {
return errors.New("don't have directive yet, can't snapshot field keys")
}
qtid := req.TableKey.QualifiedTableID()
// Confirm that this node is currently responsible for table/field/fromVersion.
var version int
if v, ok, err := api.holder.versionStore.FieldVersion(ctx, qtid, req.Field); err != nil {
return err
} else if !ok {
return errors.Errorf("field not managed by this node: %s, %s", req.TableKey, req.Field)
} else if v != req.FromVersion {
return errors.Errorf("field managed by this node is at version: %d, not: %d", v, req.FromVersion)
} else {
version = v
}
// Create the snapshot for the current version.
wrTo, err := api.FieldTranslateData(ctx, string(req.TableKey), string(req.Field))
trans, err := api.FieldTranslateData(ctx, string(req.TableKey), string(req.Field))
if err != nil {
return errors.Wrap(err, "getting index/field writeto")
return errors.Wrap(err, "getting index/field translator")
}
if err := api.snapshotReadWriter.WriteFieldKeys(ctx, qtid, req.Field, version, wrTo); err != nil {
return errors.Wrap(err, "snapshotting field keys")
// get a write tx to ensure no other writes while incrementing WL version.
wrTo, err := trans.Begin(true)
if err != nil {
return errors.Wrap(err, "beginning field translate write tx")
}
defer wrTo.Rollback()
// Increment the version of the field managed by this node.
if err := api.holder.versionStore.AddFields(ctx, qtid,
dax.NewVersionedField(req.Field, req.ToVersion),
); err != nil {
return errors.Wrap(err, "incrementing field version locally")
resource := api.serverlessStorage.GetFieldKeyResource(qtid, req.Field)
if ok, err := resource.IncrementWLVersion(); err != nil {
return errors.Wrap(err, "incrementing writelog version")
} else if !ok {
// no need to snapshot, no writes
return nil
}
// Update the cached directive on the holder.
api.holder.SetDirective(&req.Directive)
api.holder.SetDirectiveApplied(true)
// Finally, delete the log file for the previous version.
return api.writeLogWriter.DeleteFieldKeys(ctx, qtid, req.Field, req.FromVersion)
// TODO(jaffee) downgrade to read tx
err = resource.SnapshotTo(wrTo)
return errors.Wrap(err, "snapshotTo in FieldKeys")
}
type serverInfo struct {
@ -3365,9 +3303,9 @@ var methodsNormal = map[apiMethod]struct{}{
apiDeleteDataframe: {},
}
func shardInShards(i dax.ShardNum, s dax.VersionedShards) bool {
func shardInShards(i dax.ShardNum, s dax.ShardNums) bool {
for _, o := range s {
if i == o.Num {
if i == o {
return true
}
}
@ -3386,11 +3324,6 @@ type SchemaAPI interface {
DeleteField(ctx context.Context, tname dax.TableName, fname dax.FieldName) error
}
type SchemaInfoAPI interface {
IndexInfo(ctx context.Context, indexName string) (*IndexInfo, error)
FieldInfo(ctx context.Context, indexName, fieldName string) (*FieldInfo, error)
}
type ClusterNode struct {
ID string
State string
@ -3408,6 +3341,7 @@ type SystemAPI interface {
ClusterReplicaCount() int
ShardWidth() int
ClusterState() string
DataDir() string
ClusterNodes() []ClusterNode
}
@ -3476,6 +3410,10 @@ func (fsapi *FeatureBaseSystemAPI) ClusterState() string {
return string(state)
}
func (fsapi *FeatureBaseSystemAPI) DataDir() string {
return fsapi.server.dataDir
}
func (fsapi *FeatureBaseSystemAPI) ClusterNodes() []ClusterNode {
result := make([]ClusterNode, 0)

254
api_directive.go
View file

@ -9,6 +9,7 @@ import (
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/computer"
"github.com/featurebasedb/featurebase/v3/dax/storage"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/pkg/errors"
)
@ -41,6 +42,9 @@ func (api *API) ApplyDirective(ctx context.Context, d *dax.Directive) error {
if err := api.deleteAllIndexes(ctx); err != nil {
return errors.Wrap(err, "deleting all indexes")
}
if err := api.serverlessStorage.RemoveAll(); err != nil {
return errors.Wrap(err, "removing all managers")
}
// Set previousDirective to empty so the diff handles everything as new.
previousDirective = dax.Directive{}
@ -102,19 +106,19 @@ type directiveJobTableKeys struct {
directiveJobType
idx *Index
tkey dax.TableKey
partition dax.VersionedPartition
partition dax.PartitionNum
}
type directiveJobFieldKeys struct {
directiveJobType
tkey dax.TableKey
field dax.VersionedField
field dax.FieldName
}
type directiveJobShards struct {
directiveJobType
tkey dax.TableKey
shard dax.VersionedShard
shard dax.ShardNum
}
// directiveWorker is a worker in a worker pool which handles portions of a
@ -348,42 +352,38 @@ func (api *API) pushJobsTableKeys(ctx context.Context, jobs chan<- directiveJobT
}
}
func (api *API) loadTableKeys(ctx context.Context, idx *Index, tkey dax.TableKey, partition dax.VersionedPartition) error {
func (api *API) loadTableKeys(ctx context.Context, idx *Index, tkey dax.TableKey, partition dax.PartitionNum) error {
qtid := tkey.QualifiedTableID()
// Load the previous snapshot. Version 0 doesn't have a snapshot
// file; it only has log entries.
if partition.Version > 0 {
// Load partition snapshot: version - 1
previousVersion := partition.Version - 1
rc, err := api.snapshotReadWriter.ReadTableKeys(ctx, qtid, partition.Num, previousVersion)
if err != nil {
return errors.Wrap(err, "reading table keys snapshot")
}
resource := api.serverlessStorage.GetTableKeyResource(qtid, partition)
if resource.IsLocked() {
api.logger().Warnf("skipping loadTableKeys (already held) %s %d", tkey, partition)
return nil
}
// load latest snapshot
if rc, err := resource.LoadLatestSnapshot(); err != nil {
return errors.Wrap(err, "loading table key snapshot")
} else if rc != nil {
defer rc.Close()
if err := api.TranslateIndexDB(ctx, string(tkey), int(partition.Num), rc); err != nil {
if err := api.TranslateIndexDB(ctx, string(tkey), int(partition), rc); err != nil {
return errors.Wrap(err, "restoring table keys")
}
}
if err := func() error {
store := idx.TranslateStore(int(partition.Num))
reader := api.writeLogReader.TableKeyReader(ctx, qtid, partition.Num, partition.Version)
if err := reader.Open(); err != nil {
// TODO: this log can be confusing because on a create
// table, there is no log file yet, so an error is expected.
// Instead of swallowing this error, we need to check the
// error code and handle it differently. This means the
// writelogger will need to return an error indicating that
// the log file does not exist, but that that is expected.
// log.Printf("could not open log file for table: %s, partition: %d: version: %d, err: %s", table, partition.Num, partition.Version, err)
// define write log loading in a function since we have to do it
// before and after locking
loadWriteLog := func() error {
writelog, err := resource.LoadWriteLog()
if err != nil {
return errors.Wrap(err, "getting write log reader for table keys")
}
if writelog == nil {
return nil
}
reader := storage.NewTableKeyReader(qtid, partition, writelog)
defer reader.Close()
store := idx.TranslateStore(int(partition))
for msg, err := reader.Read(); err != io.EOF; msg, err = reader.Read() {
if err != nil {
return errors.Wrap(err, "reading from log reader")
@ -394,18 +394,21 @@ func (api *API) loadTableKeys(ctx context.Context, idx *Index, tkey dax.TableKey
}
}
}
return nil
}(); err != nil {
}
// 1st write log load
if err := loadWriteLog(); err != nil {
return err
}
// Set the table/partition/version in the holder.
if err := api.holder.versionStore.AddPartitions(ctx, qtid, partition); err != nil {
return errors.Wrap(err, "adding partition to sharder")
// acquire lock on this partition's keys
if err := resource.Lock(); err != nil {
return errors.Wrap(err, "locking table key partition")
}
return nil
// reload writelog in case of changes between last load and
// lock. The resource object takes care of only loading new data.
return loadWriteLog()
}
func (api *API) pushJobsFieldKeys(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
@ -423,47 +426,45 @@ func (api *API) pushJobsFieldKeys(ctx context.Context, jobs chan<- directiveJobT
}
}
func (api *API) loadFieldKeys(ctx context.Context, tkey dax.TableKey, field dax.VersionedField) error {
func (api *API) loadFieldKeys(ctx context.Context, tkey dax.TableKey, field dax.FieldName) error {
qtid := tkey.QualifiedTableID()
// Load the previous snapshot. Version 0 doesn't have a snapshot
// file; it only has log entries.
if field.Version > 0 {
// Load field snapshot: version - 1
previousVersion := field.Version - 1
rc, err := api.snapshotReadWriter.ReadFieldKeys(ctx, qtid, field.Name, previousVersion)
if err != nil {
return errors.Wrap(err, "reading field keys snapshot")
}
defer rc.Close()
resource := api.serverlessStorage.GetFieldKeyResource(qtid, field)
if resource.IsLocked() {
api.logger().Warnf("skipping loadFieldKeys (already held) %s %s", tkey, field)
return nil
}
if err := api.TranslateFieldDB(ctx, string(tkey), string(field.Name), rc); err != nil {
// load latest snapshot
if rc, err := resource.LoadLatestSnapshot(); err != nil {
return errors.Wrap(err, "loading field key snapshot")
} else if rc != nil {
defer rc.Close()
if err := api.TranslateFieldDB(ctx, string(tkey), string(field), rc); err != nil {
return errors.Wrap(err, "restoring field keys")
}
}
if err := func() error {
// define write log loading in a function since we have to do it
// before and after locking
loadWriteLog := func() error {
writelog, err := resource.LoadWriteLog()
if err != nil {
return errors.Wrap(err, "getting write log reader for field keys")
}
if writelog == nil {
return nil
}
reader := storage.NewFieldKeyReader(qtid, field, writelog)
defer reader.Close()
// Get field in order to find the translate store.
fld := api.holder.Field(string(tkey), string(field.Name))
fld := api.holder.Field(string(tkey), string(field))
if fld == nil {
log.Printf("field not found in holder: %s", field.Name)
log.Printf("field not found in holder: %s", field)
return nil
}
store := fld.TranslateStore()
reader := api.writeLogReader.FieldKeyReader(ctx, qtid, field.Name, field.Version)
if err := reader.Open(); err != nil {
// TODO: this log can be confusing because on a create
// table, there is no log file yet, so an error is expected.
// Instead of swallowing this error, we need to check the
// error code and handle it differently. This means the
// writelogger will need to return an error indicating that
// the log file does not exist, but that that is expected.
// log.Printf("could not open log file for table: %s, field: %s: version: %d, err: %s", table, field.Name, field.Version, err)
return nil
}
defer reader.Close()
for msg, err := reader.Read(); err != io.EOF; msg, err = reader.Read() {
if err != nil {
return errors.Wrap(err, "reading from log reader")
@ -474,18 +475,21 @@ func (api *API) loadFieldKeys(ctx context.Context, tkey dax.TableKey, field dax.
}
}
}
return nil
}(); err != nil {
}
// 1st write log load
if err := loadWriteLog(); err != nil {
return err
}
// Set the table/field/version in the holder.
if err := api.holder.versionStore.AddFields(ctx, qtid, field); err != nil {
return errors.Wrap(err, "adding field to sharder")
// acquire lock on this partition's keys
if err := resource.Lock(); err != nil {
return errors.Wrap(err, "locking field key partition")
}
return nil
// reload writelog in case of changes between last load and
// lock. The resource object takes care of only loading new data.
return loadWriteLog()
}
func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType, fromD, toD *dax.Directive) {
@ -506,46 +510,43 @@ func (api *API) pushJobsShards(ctx context.Context, jobs chan<- directiveJobType
}
}
func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.VersionedShard) error {
func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.ShardNum) error {
qtid := tkey.QualifiedTableID()
partition := disco.ShardToShardPartition(string(tkey), uint64(shard.Num), disco.DefaultPartitionN)
partitionNum := dax.PartitionNum(partition)
partition := dax.PartitionNum(disco.ShardToShardPartition(string(tkey), uint64(shard), disco.DefaultPartitionN))
// Load the previous snapshot. Version 0 doesn't have a snapshot
// file; it only has log entries.
if shard.Version > 0 {
// Load shard snapshot: version - 1
previousVersion := shard.Version - 1
rc, err := api.snapshotReadWriter.ReadShardData(ctx, qtid, partitionNum, shard.Num, previousVersion)
if err != nil {
return errors.Wrap(err, "reading shard data snapshot")
}
resource := api.serverlessStorage.GetShardResource(qtid, partition, shard)
if resource.IsLocked() {
api.logger().Warnf("skipping loadShard (already held) %s %d", tkey, shard)
return nil
}
if err := api.RestoreShard(ctx, string(tkey), uint64(shard.Num), rc); err != nil {
if rc, err := resource.LoadLatestSnapshot(); err != nil {
return errors.Wrap(err, "reading latest snapshot for shard")
} else if rc != nil {
defer rc.Close()
if err := api.RestoreShard(ctx, string(tkey), uint64(shard), rc); err != nil {
return errors.Wrap(err, "restoring shard data")
}
}
// WriteLog reader.
if err := func() error {
reader := api.writeLogReader.ShardReader(ctx, qtid, partitionNum, shard.Num, shard.Version)
if err := reader.Open(); err != nil {
// TODO: this log can be confusing because on a create
// table, there is no log file yet, so an error is expected.
// Instead of swallowing this error, we need to check the
// error code and handle it differently. This means the
// writelogger will need to return an error indicating that
// the log file does not exist, but that that is expected.
// log.Printf("could not open log file for table: %s, partition: %d: version: %d, shard: %d, err: %s", table, partition, shard.Version, shard.Num, err)
// define write log loading in a func because we do it twice.
loadWriteLog := func() error {
writelog, err := resource.LoadWriteLog()
if err != nil {
return errors.Wrap(err, "")
}
if writelog == nil {
return nil
}
defer reader.Close()
reader := storage.NewShardReader(qtid, partition, shard, writelog)
defer reader.Close()
for logMsg, err := reader.Read(); err != io.EOF; logMsg, err = reader.Read() {
if err != nil {
return errors.Wrap(err, "reading from log reader")
}
switch msg := logMsg.(type) {
case *computer.ImportRoaringMessage:
req := &ImportRoaringRequest{
@ -632,18 +633,21 @@ func (api *API) loadShard(ctx context.Context, tkey dax.TableKey, shard dax.Vers
}
}
}
return nil
}(); err != nil {
}
// 1st write log load
if err := loadWriteLog(); err != nil {
return err
}
// Set the table/shard/version in the holder.
if err := api.holder.versionStore.AddShards(ctx, qtid, shard); err != nil {
return errors.Wrap(err, "adding shard to sharder")
// acquire lock on this partition's keys
if err := resource.Lock(); err != nil {
return errors.Wrap(err, "locking field key partition")
}
return nil
// reload writelog in case of changes between last load and
// lock. The resource object takes care of only loading new data.
return loadWriteLog()
}
//////////////////////////////////////////////////////////////
@ -707,11 +711,11 @@ func thingsAdded[K comparable](from []K, to []K) []K {
// partitionsComparer is used to compare the differences between two maps of
// table:[]partition.
type partitionsComparer struct {
from map[dax.TableKey]dax.VersionedPartitions
to map[dax.TableKey]dax.VersionedPartitions
from map[dax.TableKey]dax.PartitionNums
to map[dax.TableKey]dax.PartitionNums
}
func newPartitionsComparer(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.TableKey]dax.VersionedPartitions) *partitionsComparer {
func newPartitionsComparer(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) *partitionsComparer {
return &partitionsComparer{
from: from,
to: to,
@ -720,23 +724,23 @@ func newPartitionsComparer(from map[dax.TableKey]dax.VersionedPartitions, to map
// added returns the partitions which are present in `to` but not in `from`. The
// results remain in the format of a map of table:[]partition.
func (p *partitionsComparer) added() map[dax.TableKey]dax.VersionedPartitions {
func (p *partitionsComparer) added() map[dax.TableKey]dax.PartitionNums {
return partitionsAdded(p.from, p.to)
}
// removed returns the partitions which are present in `from` but not in `to`.
// The results remain in the format of a map of table:[]partition.
func (p *partitionsComparer) removed() map[dax.TableKey]dax.VersionedPartitions {
func (p *partitionsComparer) removed() map[dax.TableKey]dax.PartitionNums {
return partitionsAdded(p.to, p.from)
}
// partitionsAdded returns the partitions which are present in `to` but not in `from`.
func partitionsAdded(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.TableKey]dax.VersionedPartitions) map[dax.TableKey]dax.VersionedPartitions {
func partitionsAdded(from map[dax.TableKey]dax.PartitionNums, to map[dax.TableKey]dax.PartitionNums) map[dax.TableKey]dax.PartitionNums {
if from == nil {
return to
}
added := make(map[dax.TableKey]dax.VersionedPartitions)
added := make(map[dax.TableKey]dax.PartitionNums)
for tt, tps := range to {
fps, found := from[tt]
if !found {
@ -744,7 +748,7 @@ func partitionsAdded(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.T
continue
}
addedPartitions := dax.VersionedPartitions{}
addedPartitions := dax.PartitionNums{}
for i := range tps {
var found bool
for j := range fps {
@ -768,11 +772,11 @@ func partitionsAdded(from map[dax.TableKey]dax.VersionedPartitions, to map[dax.T
// fieldsComparer is used to compare the differences between two maps of
// table:[]fieldVersion.
type fieldsComparer struct {
from map[dax.TableKey]dax.VersionedFields
to map[dax.TableKey]dax.VersionedFields
from map[dax.TableKey][]dax.FieldName
to map[dax.TableKey][]dax.FieldName
}
func newFieldsComparer(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]dax.VersionedFields) *fieldsComparer {
func newFieldsComparer(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) *fieldsComparer {
return &fieldsComparer{
from: from,
to: to,
@ -781,23 +785,23 @@ func newFieldsComparer(from map[dax.TableKey]dax.VersionedFields, to map[dax.Tab
// added returns the fields which are present in `to` but not in `from`. The
// results remain in the format of a map of table:[]field.
func (f *fieldsComparer) added() map[dax.TableKey]dax.VersionedFields {
func (f *fieldsComparer) added() map[dax.TableKey][]dax.FieldName {
return fieldsAdded(f.from, f.to)
}
// removed returns the fields which are present in `from` but not in `to`.
// The results remain in the format of a map of table:[]field.
func (f *fieldsComparer) removed() map[dax.TableKey]dax.VersionedFields {
func (f *fieldsComparer) removed() map[dax.TableKey][]dax.FieldName {
return fieldsAdded(f.to, f.from)
}
// fieldsAdded returns the fields which are present in `to` but not in `from`.
func fieldsAdded(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]dax.VersionedFields) map[dax.TableKey]dax.VersionedFields {
func fieldsAdded(from map[dax.TableKey][]dax.FieldName, to map[dax.TableKey][]dax.FieldName) map[dax.TableKey][]dax.FieldName {
if from == nil {
return to
}
added := make(map[dax.TableKey]dax.VersionedFields)
added := make(map[dax.TableKey][]dax.FieldName)
for tt, tps := range to {
fps, found := from[tt]
if !found {
@ -805,7 +809,7 @@ func fieldsAdded(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]
continue
}
addedFieldVersions := dax.VersionedFields{}
addedFieldVersions := []dax.FieldName{}
for i := range tps {
var found bool
for j := range fps {
@ -829,11 +833,11 @@ func fieldsAdded(from map[dax.TableKey]dax.VersionedFields, to map[dax.TableKey]
// shardsComparer is used to compare the differences between two maps of
// table:[]shardV.
type shardsComparer struct {
from map[dax.TableKey]dax.VersionedShards
to map[dax.TableKey]dax.VersionedShards
from map[dax.TableKey]dax.ShardNums
to map[dax.TableKey]dax.ShardNums
}
func newShardsComparer(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]dax.VersionedShards) *shardsComparer {
func newShardsComparer(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) *shardsComparer {
return &shardsComparer{
from: from,
to: to,
@ -842,23 +846,23 @@ func newShardsComparer(from map[dax.TableKey]dax.VersionedShards, to map[dax.Tab
// added returns the shards which are present in `to` but not in `from`. The
// results remain in the format of a map of table:[]shard.
func (s *shardsComparer) added() map[dax.TableKey]dax.VersionedShards {
func (s *shardsComparer) added() map[dax.TableKey]dax.ShardNums {
return shardsAdded(s.from, s.to)
}
// removed returns the shards which are present in `from` but not in `to`. The
// results remain in the format of a map of table:[]shard.
func (s *shardsComparer) removed() map[dax.TableKey]dax.VersionedShards {
func (s *shardsComparer) removed() map[dax.TableKey]dax.ShardNums {
return shardsAdded(s.to, s.from)
}
// shardsAdded returns the shards which are present in `to` but not in `from`.
func shardsAdded(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]dax.VersionedShards) map[dax.TableKey]dax.VersionedShards {
func shardsAdded(from map[dax.TableKey]dax.ShardNums, to map[dax.TableKey]dax.ShardNums) map[dax.TableKey]dax.ShardNums {
if from == nil {
return to
}
added := make(map[dax.TableKey]dax.VersionedShards)
added := make(map[dax.TableKey]dax.ShardNums)
for tt, tss := range to {
fss, found := from[tt]
if !found {
@ -866,7 +870,7 @@ func shardsAdded(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]
continue
}
addedShards := dax.VersionedShards{}
addedShards := dax.ShardNums{}
for i := range tss {
var found bool
for j := range fss {
@ -889,7 +893,7 @@ func shardsAdded(from map[dax.TableKey]dax.VersionedShards, to map[dax.TableKey]
// createTableAndFields creates the FeatureBase Tables and Fields provided in
// the dax.Directive format.
func (api *API) createTableAndFields(tbl *dax.QualifiedTable, partitions dax.VersionedPartitions) error {
func (api *API) createTableAndFields(tbl *dax.QualifiedTable, partitions dax.PartitionNums) error {
cim := &CreateIndexMessage{
Index: string(tbl.Key()),
CreatedAt: 0,

84
apply.go
View file

@ -13,8 +13,6 @@ import (
"github.com/apache/arrow/go/v10/arrow"
"github.com/apache/arrow/go/v10/arrow/array"
"github.com/apache/arrow/go/v10/arrow/memory"
"github.com/apache/arrow/go/v10/parquet/file"
"github.com/apache/arrow/go/v10/parquet/pqarrow"
"github.com/gomem/gomem/pkg/dataframe"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/tracing"
@ -220,12 +218,14 @@ func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string
if idx == nil {
return nil, newNotFoundError(ErrIndexNotFound, index)
}
fname := idx.GetDataFramePath(shard)
if _, err := os.Stat(fname + ".parquet"); os.IsNotExist(err) {
if !e.dataFrameExists(fname) {
return value.NewVector([]value.Value{}), nil
}
table, err := readTableParquet(fname)
table, err := e.getDataTable(ctx, fname, pool)
if err != nil {
return nil, err
}
@ -254,57 +254,20 @@ func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string
return context.Global("_"), nil
}
func readTableParquet(filename string) (arrow.Table, error) {
r, err := os.Open(filename + ".parquet")
if err != nil {
return nil, err
}
pf, err := file.NewParquetReader(r)
if err != nil {
return nil, err
}
reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, memory.DefaultAllocator)
if err != nil {
return nil, err
}
return reader.ReadTable(context.Background())
}
func readTableParquetCtx(ctx context.Context, filename string, mem memory.Allocator) (arrow.Table, error) {
r, err := os.Open(filename + ".parquet")
if err != nil {
return nil, err
}
pf, err := file.NewParquetReader(r)
if err != nil {
return nil, err
}
reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, mem)
if err != nil {
return nil, err
}
return reader.ReadTable(ctx)
}
// ///////////////////////////////////////////////////////
// all the ingest supporting functions
// ///////////////////////////////////////////////////////
func NewShardFile(name string) (*ShardFile, error) {
if _, err := os.Stat(name + ".parquet"); os.IsNotExist(err) {
return &ShardFile{dest: name}, nil
func NewShardFile(ctx context.Context, name string, mem memory.Allocator, e *executor) (*ShardFile, error) {
if !e.dataFrameExists(name) {
return &ShardFile{dest: name, executor: e}, nil
}
// else read in existing
table, err := readTableParquet(name)
table, err := e.getDataTable(ctx, name, mem)
if err != nil {
return nil, err
}
return &ShardFile{table: table, schema: table.Schema(), dest: name}, nil
return &ShardFile{table: table, schema: table.Schema(), dest: name, executor: e}, nil
}
type NameType struct {
@ -349,6 +312,7 @@ type ShardFile struct {
added int64
columns []interface{}
dest string
executor *executor
}
func compareSchema(s1, s2 *arrow.Schema) bool {
@ -427,7 +391,7 @@ func (sf *ShardFile) Process(cs *ChangesetRequest) error {
if err != nil {
return err
}
return os.Rename(rtemp+".parquet", sf.dest+".parquet")
return os.Rename(rtemp+sf.executor.TableExtension(), sf.dest+sf.executor.TableExtension())
}
func (sf *ShardFile) process(cs *ChangesetRequest) error {
@ -521,22 +485,9 @@ func (sf *ShardFile) Save(name string) error {
}
}
rec := array.NewRecord(sf.schema, parts, sf.beforeRows+sf.added)
df, err := dataframe.NewDataFrameFromRecord(mem, rec)
if err != nil {
return err
}
// confirm change
w, err := os.Create(name + ".parquet")
if err != nil {
return err
}
table := array.NewTableFromRecords(sf.schema, []arrow.Record{rec})
err = df.ToParquet(w, 1024)
if err != nil {
return err
}
w.Close()
return nil
return sf.executor.SaveTable(name, table, mem)
}
// TODO(twg) 2022/10/03 Not a huge fan of the global variable will look at adding to executor structure
@ -573,7 +524,8 @@ func (api *API) ApplyDataframeChangeset(ctx context.Context, index string, cs *C
mu := getDataframeWritelock(shard)
mu.Lock()
defer mu.Unlock()
shardFile, err := NewShardFile(fname)
mem := memory.NewGoAllocator()
shardFile, err := NewShardFile(ctx, fname, mem, api.server.executor)
if err != nil {
return err
}
@ -600,14 +552,16 @@ func (api *API) GetDataframeSchema(ctx context.Context, indexName string) (inter
dir, _ := os.Open(base)
files, _ := dir.Readdir(0)
parts := make([]column, 0)
mem := memory.NewGoAllocator()
for i := range files {
file := files[i]
name := file.Name()
if strings.HasSuffix(name, ".parquet") {
if api.server.executor.IsDataframeFile(name) {
// strip off the parquet extenison
name = strings.TrimSuffix(name, filepath.Ext(name))
// read the parquet file and extract the schema
table, err := readTableParquet(filepath.Join(base, name))
fname := filepath.Join(base, name)
table, err := api.server.executor.getDataTable(ctx, fname, mem)
if err != nil {
return nil, err
}

151
arrow.go
View file

@ -5,12 +5,18 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"sync"
"github.com/apache/arrow/go/v10/arrow"
"github.com/apache/arrow/go/v10/arrow/array"
"github.com/apache/arrow/go/v10/arrow/ipc"
"github.com/apache/arrow/go/v10/arrow/memory"
"github.com/apache/arrow/go/v10/parquet"
"github.com/apache/arrow/go/v10/parquet/file"
"github.com/apache/arrow/go/v10/parquet/pqarrow"
"github.com/gomem/gomem/pkg/dataframe"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/tracing"
@ -371,12 +377,14 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
if idx == nil {
return nil, newNotFoundError(ErrIndexNotFound, index)
}
fname := idx.GetDataFramePath(shard)
if _, err := os.Stat(fname + ".parquet"); os.IsNotExist(err) {
if !e.dataFrameExists(fname) {
return &basicTable{name: name}, nil
}
table, err := readTableParquetCtx(context.TODO(), fname, pool)
table, err := e.getDataTable(ctx, fname, pool)
if err != nil {
return nil, errors.Wrap(err, "arrow readTableParquet")
}
@ -403,3 +411,142 @@ func (e *executor) executeArrowShard(ctx context.Context, qcx *Qcx, index string
table.Retain()
return &basicTable{resolver: resolver, table: table, filtered: filter != nil, name: name}, nil
}
func (e *executor) dataFrameExists(fname string) bool {
if e.typeIsParquet() {
if _, err := os.Stat(fname + ".parquet"); os.IsNotExist(err) {
return false
}
return true
}
if _, err := os.Stat(fname + ".arrow"); os.IsNotExist(err) {
return false
}
return true
}
func (e *executor) getDataTable(ctx context.Context, fname string, mem memory.Allocator) (arrow.Table, error) {
if e.typeIsParquet() {
table, err := readTableParquetCtx(ctx, fname, mem)
return table, err
}
return readTableArrow(fname, mem)
}
func (e *executor) typeIsParquet() bool {
return e.datafameUseParquet
}
func (e *executor) IsDataframeFile(name string) bool {
if e.typeIsParquet() {
return strings.HasSuffix(name, ".parquet")
}
return strings.HasSuffix(name, ".arrow")
}
func (e *executor) SaveTable(name string, table arrow.Table, mem memory.Allocator) error {
if e.typeIsParquet() {
return writeTableParquet(table, name)
}
return writeTableArrow(table, name, mem)
}
func (e *executor) TableExtension() string {
if e.typeIsParquet() {
return ".parquet"
}
return ".arrow"
}
func readTableArrow(filename string, mem memory.Allocator) (arrow.Table, error) {
r, err := os.Open(filename + ".arrow")
if err != nil {
return nil, err
}
rr, err := ipc.NewFileReader(r, ipc.WithAllocator(mem))
if err != nil {
return nil, err
}
defer rr.Close()
records := make([]arrow.Record, rr.NumRecords(), rr.NumRecords())
i := 0
for {
rec, err := rr.Read()
if err == io.EOF {
break
} else if err != nil {
return nil, err
}
records[i] = rec
i++
}
records = records[:i]
table := array.NewTableFromRecords(rr.Schema(), records)
return table, nil
}
func readTableParquetCtx(ctx context.Context, filename string, mem memory.Allocator) (arrow.Table, error) {
r, err := os.Open(filename + ".parquet")
if err != nil {
return nil, err
}
defer r.Close()
pf, err := file.NewParquetReader(r)
if err != nil {
return nil, err
}
reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, mem)
if err != nil {
return nil, err
}
return reader.ReadTable(ctx)
}
func writeTableParquet(table arrow.Table, filename string) error {
f, err := os.Create(filename + ".parquet")
if err != nil {
return err
}
defer f.Close()
props := parquet.NewWriterProperties(parquet.WithDictionaryDefault(false))
arrProps := pqarrow.DefaultWriterProps()
chunkSize := 10 * 1024 * 1024
err = pqarrow.WriteTable(table, f, int64(chunkSize), props, arrProps)
if err != nil {
return err
}
f.Sync()
return nil
}
func writeTableArrow(table arrow.Table, filename string, mem memory.Allocator) error {
f, err := os.Create(filename + ".arrow")
if err != nil {
return err
}
defer f.Close()
writer, err := ipc.NewFileWriter(f, ipc.WithAllocator(mem), ipc.WithSchema(table.Schema()))
if err != nil {
panic(err)
}
chunkSize := int64(0)
tr := array.NewTableReader(table, chunkSize)
defer tr.Release()
n := 0
for tr.Next() {
arec := tr.Record()
err = writer.Write(arec)
if err != nil {
panic(err)
}
n++
}
err = writer.Close()
if err != nil {
panic(err)
}
f.Sync()
return nil
}

53
arrow_test.go Normal file
View file

@ -0,0 +1,53 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package pilosa
import (
"context"
"encoding/hex"
"math/rand"
"os"
"path/filepath"
"testing"
"github.com/apache/arrow/go/v10/arrow"
"github.com/apache/arrow/go/v10/arrow/array"
"github.com/apache/arrow/go/v10/arrow/memory"
)
func TempFileName(prefix string) string {
randBytes := make([]byte, 16)
rand.Read(randBytes)
return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes))
}
func Test_TableParquet(t *testing.T) {
// create a arrow table
schema := arrow.NewSchema(
[]arrow.Field{
{Name: "num", Type: arrow.PrimitiveTypes.Float64},
},
nil, // no metadata
)
mem := memory.NewGoAllocator()
b := array.NewRecordBuilder(mem, schema)
defer b.Release()
b.Field(0).(*array.Float64Builder).AppendValues([]float64{1.0, 1.5, 2.0}, nil)
table := array.NewTableFromRecords(schema, []arrow.Record{b.NewRecord()})
defer table.Release()
fileName := TempFileName("pq-")
// save it as a parquet file
err := writeTableParquet(table, fileName)
if err != nil {
t.Fatal(err)
}
defer os.Remove(fileName)
// read it back in and compare the result
got, err := readTableParquetCtx(context.Background(), fileName, mem)
if err != nil {
t.Fatalf("readTableParquetCtx() error = %v", err)
}
if got.NumCols() != table.NumCols() {
t.Errorf("got:%v expected:%v", got.NumCols(), table.NumCols())
}
}

2
audit_test.go
View file

@ -7,7 +7,7 @@ import (
"os"
"reflect"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/testhook"
)

44
batch/batch.go
View file

@ -552,7 +552,6 @@ func (b *Batch) Add(rec Row) error {
// empty string is not a valid value at this point (Pilosa refuses to translate it)
if val == "" { //
b.rowIDs[i] = append(rowIDs, nilSentinel)
} else if rowID, ok := b.getRowTranslation(field.Name, val); ok {
b.rowIDs[i] = append(rowIDs, rowID)
} else {
@ -742,23 +741,27 @@ var ErrBatchNowStale = errors.New("batch is stale and needs to be imported (howe
func (b *Batch) Import() error {
ctx := context.Background()
start := time.Now()
trns, err := b.importer.StartTransaction(ctx, "", b.prevDuration*10, false, time.Hour)
if err != nil {
return errors.Wrap(err, "starting transaction")
if !b.useShardTransactionalEndpoint {
trns, err := b.importer.StartTransaction(ctx, "", b.prevDuration*10, false, time.Hour)
if err != nil {
return errors.Wrap(err, "starting transaction")
}
defer func() {
if trns != nil {
if trnsl, err := b.importer.FinishTransaction(ctx, trns.ID); err != nil {
b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
}
}
}()
}
defer func() {
if trns != nil {
if trnsl, err := b.importer.FinishTransaction(ctx, trns.ID); err != nil {
b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
}
}
b.importer.StatsTiming(MetricBatchImportDurationSeconds, time.Since(start), 1.0)
}()
size := len(b.ids)
transStart := time.Now()
// first we need to translate the toTranslate, then fill out the missing row IDs
err = b.doTranslation()
err := b.doTranslation()
if err != nil {
return errors.Wrap(err, "doing Translation")
}
@ -1313,20 +1316,15 @@ func (b *Batch) makeFragments(frags, clearFrags fragments) (fragments, fragments
shardWidth := b.shardWidth()
emptyClearRows := make(map[int]uint64)
// create _exists fragments if needed
// TODO(tlt): maybe make this a separate flag for backward compatibility?
// (because dax.Table doesn't have this).
//if b.index.Options.TrackExistence {
if true {
var curBM *roaring.Bitmap
curShard := ^uint64(0) // impossible sentinel value for shard.
for _, col := range b.ids {
if col/shardWidth != curShard {
curShard = col / shardWidth
curBM = frags.GetOrCreate(curShard, "_exists", "")
}
curBM.DirectAdd(col % shardWidth)
// create _exists fragments
var curBM *roaring.Bitmap
curShard := ^uint64(0) // impossible sentinel value for shard.
for _, col := range b.ids {
if col/shardWidth != curShard {
curShard = col / shardWidth
curBM = frags.GetOrCreate(curShard, "_exists", "")
}
curBM.DirectAdd(col % shardWidth)
}
for i, rowIDs := range b.rowIDs {

262
bufferpool/bufferpool.go Normal file
View file

@ -0,0 +1,262 @@
package bufferpool
import (
"errors"
"fmt"
"sync"
)
// FrameID is the type for frame id
type FrameID int
// PageID is the type for page id
type PageID int
var pageSyncPool = sync.Pool{
New: func() any {
pg := new(Page)
pg.id = PageID(INVALID_PAGE)
pg.isDirty = false
pg.pinCount = 0
return pg
},
}
// BufferPool represents a buffer pool of pages
type BufferPool struct {
// the underlying storage
diskManager DiskManager
// the actual pages in the buffer pool
pages []*Page
// the replacer that will elect replacements when buffer pool is full
replacer *ClockReplacer
// the list of free frames
freeList []FrameID
// the map of frames to page ids to frame ids
// frame ids are the offset into pages
// if you ask the pool for page 673, this will know at
// what offset in pages page 673 will exist
pageTable map[PageID]FrameID
}
// TODO(pok) implement a lazy writer
// * if free list is 'low' then
// * increase size of cache if there is physical memory available
// * write out old pages and boot them from the cache to increase free list
// TODO(pok) implement a checkpoint that scans the pool and writes out dirty pages every
// minute or so
// NewBufferPool returns a buffer pool
func NewBufferPool(maxSize int, diskManager DiskManager) *BufferPool {
freeList := make([]FrameID, 0)
pages := make([]*Page, maxSize)
for i := 0; i < maxSize; i++ {
frameNumber := FrameID(i)
freeList = append(freeList, frameNumber)
}
clockReplacer := NewClockReplacer(maxSize)
return &BufferPool{
diskManager: diskManager,
pages: pages,
replacer: clockReplacer,
freeList: freeList,
pageTable: make(map[PageID]FrameID),
}
}
// Dumps all the pages in the buffer pool
func (b *BufferPool) Dump() {
fmt.Println()
fmt.Printf("------------------------------------------------------------------------------------------\n")
fmt.Printf("BUFFER POOL\n")
for _, p := range b.pages {
if p != nil {
p.Dump("")
}
}
fmt.Printf("------------------------------------------------------------------------------------------\n")
fmt.Println()
}
// FetchPage fetches the requested page from the buffer pool.
func (b *BufferPool) FetchPage(pageID PageID) (*Page, error) {
// if it is in buffer pool already then just return it
if frameID, ok := b.pageTable[pageID]; ok {
page := b.pages[frameID]
page.pinCount++
b.replacer.Pin(frameID)
return page, nil
}
// not in the buffer pool so try the free list or
// the replacer will vote a page off the island
frameID, isFromFreeList, err := b.getFrameID()
if err != nil {
return nil, err
}
if !isFromFreeList {
// if it didn't come from the freelist then
// remove page from current frame, writing it out if dirty
currentPage := b.pages[frameID]
if currentPage != nil {
if currentPage.isDirty {
b.diskManager.WritePage(currentPage)
}
delete(b.pageTable, currentPage.id)
}
}
// if we got to here, sorry, have to do an I/O
page, err := b.diskManager.ReadPage(pageID)
if err != nil {
return nil, err
}
page.pinCount = 1
b.pageTable[pageID] = frameID
pageSyncPool.Put(b.pages[frameID])
b.pages[frameID] = page
b.replacer.Pin(frameID)
return page, nil
}
// UnpinPage unpins the target page from the buffer pool
func (b *BufferPool) UnpinPage(pageID PageID) error {
if frameID, ok := b.pageTable[pageID]; ok {
page := b.pages[frameID]
page.DecPinCount()
if page.pinCount <= 0 {
b.replacer.Unpin(frameID)
}
return nil
}
return errors.New("could not find page")
}
// FlushPage Flushes the target page to disk
func (b *BufferPool) FlushPage(pageID PageID) bool {
if frameID, ok := b.pageTable[pageID]; ok {
page := b.pages[frameID]
page.DecPinCount()
b.diskManager.WritePage(page)
page.isDirty = false
return true
}
return false
}
// NewPage allocates a new page in the buffer pool with the disk manager help
func (b *BufferPool) NewPage() (*Page, error) {
// get a free frame
frameID, isFromFreeList, err := b.getFrameID()
if err != nil {
return nil, err
}
if !isFromFreeList {
// remove page from current frame
currentPage := b.pages[frameID]
if currentPage != nil {
if currentPage.isDirty {
b.diskManager.WritePage(currentPage)
}
delete(b.pageTable, currentPage.id)
}
}
// allocates new page
pageID, err := b.diskManager.AllocatePage()
if err != nil {
return nil, err
}
page := &Page{pageID, 1, false, [PAGE_SIZE]byte{}}
page.WritePageNumber(int32(pageID))
page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
page.WriteNextPointer(int32(INVALID_PAGE))
page.WritePrevPointer(int32(INVALID_PAGE))
// update the frame table
b.pageTable[pageID] = frameID
pageSyncPool.Put(b.pages[frameID])
b.pages[frameID] = page
return page, nil
}
// ScratchPage returns a page outside the buffer pool - do not use if you intend the page
// to be in the buffer pool (use NewPage() for that)
// ScratchPage is intended to be used in cases where you need the Page primitives
// and will copy the scratch page back over a real page later
func (b *BufferPool) ScratchPage() *Page {
page := &Page{
id: PageID(INVALID_PAGE),
pinCount: 0,
isDirty: false,
data: [PAGE_SIZE]byte{},
}
page.WritePageNumber(int32(INVALID_PAGE))
page.WriteFreeSpaceOffset(int16(PAGE_SIZE))
page.WriteNextPointer(int32(INVALID_PAGE))
page.WritePrevPointer(int32(INVALID_PAGE))
return page
}
// DeletePage deletes a page from the buffer pool
func (b *BufferPool) DeletePage(pageID PageID) error {
var frameID FrameID
var ok bool
if frameID, ok = b.pageTable[pageID]; !ok {
return nil
}
page := b.pages[frameID]
if page.pinCount > 0 {
return errors.New("pin count greater than 0")
}
delete(b.pageTable, page.id)
b.replacer.Pin(frameID)
b.diskManager.DeallocatePage(pageID)
b.freeList = append(b.freeList, frameID)
return nil
}
// FlushAllpages flushes all the pages in the buffer pool to disk
// Yeah, never call this unless you know what you are doing
func (b *BufferPool) FlushAllpages() {
for pageID := range b.pageTable {
b.FlushPage(pageID)
}
}
func (b *BufferPool) getFrameID() (FrameID, bool, error) {
if len(b.freeList) > 0 {
frameID, newFreeList := b.freeList[0], b.freeList[1:]
b.freeList = newFreeList
return frameID, true, nil
}
victim, err := b.replacer.Victim()
return victim, false, err
}
// OnDiskSize exposes the on disk size of the backing store
// behind this buffer pool
func (b *BufferPool) OnDiskSize() int64 {
return b.diskManager.FileSize()
}
// Close closes the buffer pool
func (b *BufferPool) Close() {
b.diskManager.Close()
}

93
bufferpool/circularlist.go Normal file
View file

@ -0,0 +1,93 @@
package bufferpool
import (
"errors"
)
type circularListNode struct {
key interface{}
value interface{}
next *circularListNode
prev *circularListNode
}
type circularList struct {
head *circularListNode
tail *circularListNode
size int
capacity int
}
func newCircularList(maxSize int) *circularList {
return &circularList{nil, nil, 0, maxSize}
}
func (c *circularList) find(key interface{}) *circularListNode {
ptr := c.head
for i := 0; i < c.size; i++ {
if ptr.key == key {
return ptr
}
ptr = ptr.next
}
return nil
}
func (c *circularList) hasKey(key interface{}) bool {
return c.find(key) != nil
}
func (c *circularList) insert(key interface{}, value interface{}) error {
if c.size == c.capacity {
return errors.New("list is full")
}
newNode := &circularListNode{key, value, nil, nil}
if c.size == 0 {
newNode.next = newNode
newNode.prev = newNode
c.head = newNode
c.tail = newNode
c.size++
return nil
}
node := c.find(key)
if node != nil {
node.value = value
return nil
}
newNode.next = c.head
newNode.prev = c.tail
c.tail.next = newNode
if c.head == c.tail {
c.head.next = newNode
}
c.tail = newNode
c.head.prev = c.tail
c.size++
return nil
}
func (c *circularList) remove(key interface{}) {
node := c.find(key)
if node == nil {
return
}
if c.size == 1 {
c.head = nil
c.tail = nil
c.size--
return
}
if node == c.head {
c.head = c.head.next
}
if node == c.tail {
c.tail = c.tail.prev
}
node.next.prev = node.prev
node.prev.next = node.next
c.size--
}

64
bufferpool/clockreplacer.go Normal file
View file

@ -0,0 +1,64 @@
package bufferpool
import "errors"
// ClockReplacer implements a clock replacer algorithm
type ClockReplacer struct {
cList *circularList
clockHand **circularListNode
}
// NewClockReplacer instantiates a new clock replacer
func NewClockReplacer(poolSize int) *ClockReplacer {
cList := newCircularList(poolSize)
return &ClockReplacer{cList, &cList.head}
}
// Victim removes the victim frame as defined by the replacement policy
func (c *ClockReplacer) Victim() (FrameID, error) {
if c.cList.size == 0 {
return FrameID(INVALID_PAGE), errors.New("no victims available")
}
var victimFrameID FrameID
currentNode := (*c.clockHand)
for {
if currentNode.value.(bool) {
currentNode.value = false
c.clockHand = &currentNode.next
} else {
frameID := currentNode.key.(FrameID)
victimFrameID = frameID
c.clockHand = &currentNode.next
c.cList.remove(currentNode.key)
return victimFrameID, nil
}
}
}
// Unpin unpins a frame, indicating that it can now be victimized
func (c *ClockReplacer) Unpin(id FrameID) {
if !c.cList.hasKey(id) {
c.cList.insert(id, true)
if c.cList.size == 1 {
c.clockHand = &c.cList.head
}
}
}
// Pin pins a frame, indicating that it should not be victimized until it is unpinned
func (c *ClockReplacer) Pin(id FrameID) {
node := c.cList.find(id)
if node == nil {
return
}
if (*c.clockHand) == node {
c.clockHand = &(*c.clockHand).next
}
c.cList.remove(id)
}
// Size returns the size of the clock
func (c *ClockReplacer) Size() int {
return c.cList.size
}

21
bufferpool/diskmanager.go Normal file
View file

@ -0,0 +1,21 @@
package bufferpool
// DiskManager is responsible for interacting with disk
type DiskManager interface {
// reads a page from the disk
ReadPage(PageID) (*Page, error)
// writes a page to the disk
WritePage(*Page) error
// allocates a page
AllocatePage() (PageID, error)
// deallocates a page
DeallocatePage(PageID) error
// returns on disk file size
FileSize() int64
// closes and does any clean up
Close()
}

162
bufferpool/inmemdiskmanager.go Normal file
View file

@ -0,0 +1,162 @@
package bufferpool
import (
"errors"
"fmt"
"os"
uuid "github.com/satori/go.uuid"
)
// InMemDiskSpillingDiskManager is a memory implementation for a DiskManager interface
// that can spill to disk when a threshold is reached
type InMemDiskSpillingDiskManager struct {
// tracks the number of pages
numPages int
onDiskPages int
// tracks the number of pages we can consume before spilling
thresholdPages int
hasSpilled *struct{}
fd *os.File
// the data buffer
data []byte
}
// NewInMemDiskSpillingDiskManager returns a in-memory version of disk manager
func NewInMemDiskSpillingDiskManager(thresholdPages int) *InMemDiskSpillingDiskManager {
dm := &InMemDiskSpillingDiskManager{
numPages: 0,
thresholdPages: thresholdPages,
data: make([]byte, 0),
}
return dm
}
// ReadPage reads a page from pages
func (d *InMemDiskSpillingDiskManager) ReadPage(pageID PageID) (*Page, error) {
// check we're not asking for page out of range
if pageID < 0 || int(pageID) >= d.numPages {
return nil, errors.New("page not found")
}
// check that the offset is within range
offset := int(pageID) * PAGE_SIZE
var page = pageSyncPool.Get().(*Page)
// we have to do this stupid check because if -cpuprofile is set for go test, this
// the previous line return a weird nil-ish thing...
if page == (*Page)(nil) {
page = pageSyncPool.New().(*Page)
}
page.id = pageID
// do the read
if d.hasSpilled == nil {
if offset+PAGE_SIZE > len(d.data) {
return nil, errors.New("offset out of range")
}
b := copy(page.data[:], d.data[offset:offset+PAGE_SIZE])
fmt.Printf("bytes read: %d", b)
} else {
var err error
if offset+PAGE_SIZE > d.numPages*PAGE_SIZE {
return nil, errors.New("offset out of range")
}
_, err = d.fd.ReadAt(page.data[:], int64(offset))
if err != nil {
return nil, err
}
}
return page, nil
}
// WritePage writes a page in memory to pages
func (d *InMemDiskSpillingDiskManager) WritePage(page *Page) error {
// make sure the offset is sensible
offset := int(page.ID()) * PAGE_SIZE
// do the write
if d.hasSpilled == nil {
if offset+PAGE_SIZE > len(d.data) {
return errors.New("offset out of range")
}
copy(d.data[offset:], page.data[:])
} else {
var err error
if offset+PAGE_SIZE > d.numPages*PAGE_SIZE {
return errors.New("offset out of range")
}
_, err = d.fd.WriteAt(page.data[:], int64(offset))
if err != nil {
return err
}
// err = d.fd.Sync()
// if err != nil {
// return err
// }
}
return nil
}
// AllocatePage allocates a page and returns the page number
func (d *InMemDiskSpillingDiskManager) AllocatePage() (PageID, error) {
d.numPages = d.numPages + 1
pageID := PageID(d.numPages - 1)
if d.hasSpilled == nil {
// we have not spilled (yet), so make storage bigger
newData := make([]byte, PAGE_SIZE)
d.data = append(d.data, newData...)
// check to see if we need to spill
if d.numPages > d.thresholdPages {
fileUUID, err := uuid.NewV4()
if err != nil {
return PageID(INVALID_PAGE), err
}
// TODO(pok) we should try to tell the OS not to cache this file
d.fd, err = os.CreateTemp("", fmt.Sprintf("fb-ehash-%s", fileUUID.String()))
if err != nil {
return PageID(INVALID_PAGE), err
}
_, err = d.fd.WriteAt(d.data, 0)
if err != nil {
return PageID(INVALID_PAGE), err
}
d.data = []byte{}
d.hasSpilled = &struct{}{}
}
} else {
if d.numPages >= d.onDiskPages {
// grow the file by a chunk - 512 pages
d.onDiskPages += 512
var err error
size := int64(d.onDiskPages * PAGE_SIZE)
_, err = d.fd.WriteAt([]byte{0}, size-1)
if err != nil {
return PageID(INVALID_PAGE), err
}
}
}
return pageID, nil
}
// DeallocatePage removes page from disk
func (d *InMemDiskSpillingDiskManager) DeallocatePage(pageID PageID) error {
// nothing to do right now
return nil
}
func (d *InMemDiskSpillingDiskManager) FileSize() int64 {
return int64(len(d.data))
}
func (d *InMemDiskSpillingDiskManager) Close() {
// close and delete the file if we spilled
if d.fd != nil {
_ = d.fd.Close()
os.Remove(d.fd.Name())
}
}

371
bufferpool/page.go Normal file
View file

@ -0,0 +1,371 @@
package bufferpool
import (
"encoding/binary"
"errors"
"fmt"
)
const PAGE_SIZE int = 8192
const INVALID_PAGE int = -1
const PAGE_TYPE_BTREE_INTERNAL = 10
const PAGE_TYPE_BTREE_LEAF = 11
const PAGE_TYPE_HASH_TABLE = 12
// PAGE
// page size 8192 bytes
// byte aligned, big endian
// |====================================================|
// | offset | length | |
// |----------------------------------------------------|
// | header |
// |====================================================|
// | 0 | 4 | pageNumber (int32) |
// | 4 | 2 | pageType (int16) |
// | 6 | 2 | slotCount (int16) |
// | 8 | 2 | localDepth (int16) |
// | 10 | 2 | freeSpaceOffset (int16) |
// | 12 | 4 | prevPointer (int32) |
// | 16 | 4 | nextPointer (int32) |
// |====================================================|
// | <start of slot array 1..slotCount> |
// |----------------------------------------------------|
// | 20 | slotcount | slot entry is 2 int16 |
// | | * slotwidth | values (payloadOffset, |
// | | * #slots | payloadLength) |
// |----------------------------------------------------|
// | <free space> |
// |----------------------------------------------------|
// | <payload starting at freeSpaceOffset> |
// | payload entries are keylength (int16), key bytes, |
// | payload length (int32), payload bytes |
// |====================================================|
const PAGE_NUMBER_OFFSET = 0 // offset 0, length 4, end 4
const PAGE_TYPE_OFFSET = 4 // offset 4, length 2, end 6
const PAGE_SLOT_COUNT_OFFSET = 6 // offset 6, length 2, end 8
const PAGE_LOCAL_DEPTH_OFFSET = 8 // offset 8, length 2, end 10
const PAGE_FREE_SPACE_OFFSET = 10 // offset 10, length 2, end 12
const PAGE_PREV_POINTER_OFFSET = 12 // offset 12, length 4, end 16
const PAGE_NEXT_POINTER_OFFSET = 16 // offset 16, length 4, end 20
const PAGE_SLOTS_START_OFFSET = 20 // offset 20
// page slots
//
// key offset int16 //offset 0, length 2, end 2
// value offset int16 //offset 2, length 2, end 4
const PAGE_SLOT_LENGTH = 4
// Page represents a page on disk
type Page struct {
id PageID
pinCount int
isDirty bool
data [PAGE_SIZE]byte
}
type PageSlot struct {
KeyOffset int16
ValueOffset int16
}
func (s *PageSlot) KeyBytes(page *Page) []byte {
offset := s.KeyOffset
keyLen := int16(binary.BigEndian.Uint16(page.data[offset:]))
offset += 2
result := make([]byte, keyLen)
copy(result, page.data[offset:offset+keyLen])
return result
}
func (s *PageSlot) KeyAsInt(page *Page) int32 {
return int32(binary.BigEndian.Uint32(page.data[s.KeyOffset+2:]))
}
func (s *PageSlot) ValueBytes(page *Page) []byte {
offset := s.ValueOffset
valueLen := int32(binary.BigEndian.Uint32(page.data[offset:]))
offset += 4
result := make([]byte, valueLen)
copy(result, page.data[offset:int32(offset)+valueLen])
return result
}
func (s *PageSlot) ValueAsPagePointer(page *Page) int32 {
return int32(binary.BigEndian.Uint32(page.data[s.ValueOffset+4:]))
}
type PageChunk struct {
KeyLength int16
KeyBytes []byte
// TODO(pok) ValueBytes can be up to int32 long
// this requires an overflow page mechanism, that is not implemented
// yet, so be aware of this when storing stuff...
ValueLength int32
ValueBytes []byte
}
func (pc *PageChunk) Length() int {
return 2 + len(pc.KeyBytes) + 4 + len(pc.ValueBytes)
}
func (pc *PageChunk) ComputeKeyOffset(pageOffset int) int {
return pageOffset
}
func (pc *PageChunk) ComputeValueOffset(pageOffset int) int {
return pageOffset + 2 + len(pc.KeyBytes)
}
func (p *Page) WritePageNumber(pageNumber int32) {
p.id = PageID(pageNumber)
binary.BigEndian.PutUint32(p.data[PAGE_NUMBER_OFFSET:], uint32(pageNumber))
p.isDirty = true
}
func (p *Page) ReadPageNumber() int {
return int(binary.BigEndian.Uint32(p.data[PAGE_NUMBER_OFFSET:]))
}
func (p *Page) WritePageType(pageType int16) {
binary.BigEndian.PutUint16(p.data[PAGE_TYPE_OFFSET:], uint16(pageType))
p.isDirty = true
}
func (p *Page) ReadPageType() int16 {
return int16(binary.BigEndian.Uint16(p.data[PAGE_TYPE_OFFSET:]))
}
func (p *Page) WriteSlotCount(slotCount int16) {
binary.BigEndian.PutUint16(p.data[PAGE_SLOT_COUNT_OFFSET:], uint16(slotCount))
p.isDirty = true
}
func (p *Page) ReadSlotCount() int16 {
return int16(binary.BigEndian.Uint16(p.data[PAGE_SLOT_COUNT_OFFSET:]))
}
func (p *Page) WriteLocalDepth(localDepth int16) {
binary.BigEndian.PutUint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:], uint16(localDepth))
p.isDirty = true
}
func (p *Page) ReadLocalDepth() int16 {
return int16(binary.BigEndian.Uint16(p.data[PAGE_LOCAL_DEPTH_OFFSET:]))
}
func (p *Page) ReadSlot(slot int16) PageSlot {
offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
keyOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
offset += 2
valueOffset := int16(binary.BigEndian.Uint16(p.data[offset:]))
return PageSlot{
KeyOffset: keyOffset,
ValueOffset: valueOffset,
}
}
func (p *Page) WriteSlot(slot int16, value PageSlot) {
offset := PAGE_SLOTS_START_OFFSET + PAGE_SLOT_LENGTH*slot
binary.BigEndian.PutUint16(p.data[offset:], uint16(value.KeyOffset))
offset += 2
binary.BigEndian.PutUint16(p.data[offset:], uint16(value.ValueOffset))
}
func (p *Page) WriteFreeSpaceOffset(offset int16) {
binary.BigEndian.PutUint16(p.data[PAGE_FREE_SPACE_OFFSET:], uint16(offset))
p.isDirty = true
}
func (p *Page) ReadFreeSpaceOffset() int16 {
return int16(binary.BigEndian.Uint16(p.data[PAGE_FREE_SPACE_OFFSET:]))
}
func (p *Page) WritePrevPointer(prevPointer int32) {
binary.BigEndian.PutUint32(p.data[PAGE_PREV_POINTER_OFFSET:], uint32(prevPointer))
p.isDirty = true
}
func (p *Page) ReadPrevPointer() int {
return int(binary.BigEndian.Uint32(p.data[PAGE_PREV_POINTER_OFFSET:]))
}
func (p *Page) WriteNextPointer(nextPointer int32) {
binary.BigEndian.PutUint32(p.data[PAGE_NEXT_POINTER_OFFSET:], uint32(nextPointer))
p.isDirty = true
}
func (p *Page) ReadNextPointer() int {
return int(binary.BigEndian.Uint32(p.data[PAGE_NEXT_POINTER_OFFSET:]))
}
func (p *Page) WriteChunk(offset int16, chunk PageChunk) {
binary.BigEndian.PutUint16(p.data[offset:], uint16(chunk.KeyLength))
offset += 2
copy(p.data[offset:], chunk.KeyBytes)
offset += int16(len(chunk.KeyBytes))
binary.BigEndian.PutUint32(p.data[offset:], uint32(chunk.ValueLength))
offset += 4
copy(p.data[offset:], chunk.ValueBytes)
p.isDirty = true
}
func (p *Page) ReadChunk(offset int16) PageChunk {
keyLen := int16(binary.BigEndian.Uint16(p.data[offset:]))
offset += 2
keyBytes := make([]byte, keyLen)
copy(keyBytes, p.data[offset:offset+keyLen])
offset += keyLen
valueLen := int32(binary.BigEndian.Uint32(p.data[offset:]))
offset += 4
valueBytes := make([]byte, valueLen)
copy(valueBytes, p.data[offset:int32(offset)+valueLen])
return PageChunk{
KeyLength: keyLen,
KeyBytes: keyBytes,
ValueLength: valueLen,
ValueBytes: valueBytes,
}
}
func (p *Page) FreeSpace() int16 {
freeSpaceOffset := p.ReadFreeSpaceOffset()
freespace := freeSpaceOffset - (p.ReadSlotCount()*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET)
return freespace
}
func (p *Page) WriteKeyValueInSlot(slotNumber int16, key []byte, value []byte) error {
freeSpaceOffset := p.ReadFreeSpaceOffset()
// build a chunk
chunk := PageChunk{
KeyLength: int16(len(key)),
KeyBytes: key,
ValueLength: int32(len(value)),
ValueBytes: value,
}
// compute the new free space offset
freeSpaceOffset -= int16(chunk.Length())
// check we won't blow free space on page
slotCount := p.ReadSlotCount()
slotEndOffset := slotCount*PAGE_SLOT_LENGTH + PAGE_SLOT_LENGTH + PAGE_SLOTS_START_OFFSET
// DEBUG!!
//fmt.Printf("freeSpaceOffset: %d, slotCount: %d, slotCount*4 + 4 + 20: %d, freeSpace: %d\n", freeSpaceOffset, slotCount, slotEndOffset, freeSpaceOffset-slotEndOffset)
if freeSpaceOffset-slotEndOffset <= 0 {
return errors.New("page is full")
}
keyOffset := chunk.ComputeKeyOffset(int(freeSpaceOffset))
valueOffset := chunk.ComputeValueOffset(int(freeSpaceOffset))
p.WriteChunk(freeSpaceOffset, chunk)
// update the free space offset
p.WriteFreeSpaceOffset(int16(freeSpaceOffset))
// make a slot
slot := PageSlot{
KeyOffset: int16(keyOffset),
ValueOffset: int16(valueOffset),
}
// write the slot
p.WriteSlot(slotNumber, slot)
return nil
}
func (p *Page) WritePage(page *Page) {
// copy everything but pageNumber & pageType
offset := PAGE_SLOT_COUNT_OFFSET
copy(page.data[offset:], p.data[offset:offset+PAGE_SIZE-offset])
}
func (p *Page) PinCount() int {
return p.pinCount
}
func (p *Page) ID() PageID {
return p.id
}
func (p *Page) DecPinCount() {
if p.pinCount > 0 {
p.pinCount--
}
}
type PageSlotIterator struct {
page *Page
slotCount int16
cursor int16
}
func NewPageSlotIterator(page *Page, fromSlot int16) *PageSlotIterator {
i := &PageSlotIterator{
page: page,
slotCount: page.ReadSlotCount(),
cursor: fromSlot,
}
return i
}
func (i *PageSlotIterator) Next() *PageSlot {
if i.cursor < i.slotCount {
s := i.page.ReadSlot(i.cursor)
i.cursor++
return &s
}
return nil
}
func (i *PageSlotIterator) Cursor() int16 {
return i.cursor
}
func (pg *Page) Dump(label string) {
indent := 0
if len(label) > 0 {
fmt.Printf("%s%s:\n", fmt.Sprintf("%*s", indent, ""), label)
indent += 4
}
pageType := pg.ReadPageType()
fmt.Printf("%sPAGE(%d) pageType: %d slotCount: %d, prevPtr: %d, nextPtr: %d\n", fmt.Sprintf("%*s", indent, ""), pg.ID(), pageType, pg.ReadSlotCount(), pg.ReadPrevPointer(), pg.ReadNextPointer())
fmt.Printf("%sKEYS: -->\n", fmt.Sprintf("%*s", indent, ""))
indent += 4
// get the keys off the page
keys := make([]int, 0)
pointers := make([]int, 0)
iter := NewPageSlotIterator(pg, 0)
for {
ps := iter.Next()
if ps == nil {
break
}
keys = append(keys, int(ps.KeyAsInt(pg)))
if pageType == /*nodeTypeInternal*/ 10 {
pointers = append(pointers, int(ps.ValueAsPagePointer(pg)))
}
}
if pageType == /*nodeTypeLeaf*/ 11 {
for _, key := range keys {
fmt.Printf("%s(%d)\n", fmt.Sprintf("%*s", indent, ""), key)
}
} else {
for idx, key := range keys {
ptr := pointers[idx]
fmt.Printf("%s(%d, %d)\n", fmt.Sprintf("%*s", indent, ""), key, ptr)
}
ptr := pg.ReadNextPointer()
fmt.Printf("%s(-->, %d)\n", fmt.Sprintf("%*s", indent, ""), ptr)
}
}

2
cache_test.go
View file

@ -6,7 +6,7 @@ import (
"reflect"
"testing"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3"
)
// Ensure cache stays constrained to its configured size.

10
client/client.go
View file

@ -1842,15 +1842,5 @@ func (c *Client) ApplyDataframeChangeset(indexName string, cr *pilosa.ChangesetR
})
}
err = eg.Wait()
// status, body, err := c.HTTPRequest(http.MethodPost, path, buffer.Bytes(), headers)
/*
var result map[string]interface{}
err = json.Unmarshal(body, &result)
if err != nil {
return nil, errors.Wrap(err, "unmarshalling response")
}
*/
return nil, err
}

72
cluster.go
View file

@ -4,6 +4,7 @@ package pilosa
import (
"context"
"encoding/json"
"fmt"
"log"
"sync"
@ -11,6 +12,7 @@ import (
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/computer"
"github.com/featurebasedb/featurebase/v3/dax/storage"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/featurebasedb/featurebase/v3/roaring"
@ -72,8 +74,7 @@ type cluster struct { // nolint: maligned
partitionAssigner string
writeLogWriter computer.WriteLogWriter
versionStore dax.VersionStore
serverlessStorage *storage.ResourceManager
// isComputeNode is set to true if this node is running as a DAX compute
// node.
@ -100,8 +101,6 @@ func newCluster() *cluster {
disCo: disco.NopDisCo,
noder: disco.NewEmptyLocalNoder(),
writeLogWriter: computer.NewNopWriteLogWriter(),
}
}
@ -318,6 +317,44 @@ func (c *cluster) findFieldKeys(ctx context.Context, field *Field, keys ...strin
return translations, nil
}
func (c *cluster) appendFieldKeysWriteLog(ctx context.Context, qtid dax.QualifiedTableID, fieldName dax.FieldName, translations map[string]uint64) error {
// TODO move marshaling somewhere more centralized and less... explicitly json-y
msg := computer.FieldKeyMap{
TableKey: qtid.Key(),
Field: fieldName,
StringToID: translations,
}
b, err := json.Marshal(msg)
if err != nil {
return errors.Wrap(err, "marshalling field key map to json")
}
resource := c.serverlessStorage.GetFieldKeyResource(qtid, fieldName)
err = resource.Append(b)
if err != nil {
return errors.Wrap(err, "appending field keys")
}
return nil
}
func (c *cluster) appendTableKeysWriteLog(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, translations map[string]uint64) error {
msg := computer.PartitionKeyMap{
TableKey: qtid.Key(),
Partition: partition,
StringToID: translations,
}
b, err := json.Marshal(msg)
if err != nil {
return errors.Wrap(err, "marshalling partition key map to json")
}
resource := c.serverlessStorage.GetTableKeyResource(qtid, partition)
return errors.Wrap(resource.Append(b), "appending table keys")
}
func (c *cluster) createFieldKeys(ctx context.Context, field *Field, keys ...string) (map[string]uint64, error) {
if idx := field.ForeignIndex(); idx != "" {
// The field uses foreign index keys.
@ -350,17 +387,9 @@ func (c *cluster) createFieldKeys(ctx context.Context, field *Field, keys ...str
tkey := dax.TableKey(field.Index())
qtid := tkey.QualifiedTableID()
fieldName := dax.FieldName(field.Name())
// Get the current version for field.
version, found, err := c.versionStore.FieldVersion(ctx, qtid, fieldName)
err = c.appendFieldKeysWriteLog(ctx, qtid, fieldName, translations)
if err != nil {
return nil, errors.Wrap(err, "getting field version")
} else if !found {
return nil, errors.Errorf("no version found for table(%s) field(%s)", qtid, fieldName)
}
if err := c.writeLogWriter.CreateFieldKeys(ctx, qtid, fieldName, version, translations); err != nil {
return nil, errors.Errorf("logging field(%s/%s) keys(%v)", field.Index(), field.Name(), keys)
return nil, errors.Wrap(err, "appending to write log")
}
return translations, nil
@ -754,16 +783,7 @@ func (c *cluster) createIndexKeys(ctx context.Context, indexName string, keys ..
tkey := dax.TableKey(idx.Name())
qtid := tkey.QualifiedTableID()
partitionNum := dax.PartitionNum(partitionID)
// Get the current version for partition.
version, found, err := c.versionStore.PartitionVersion(ctx, qtid, partitionNum)
if err != nil {
return errors.Wrap(err, "getting partition version")
} else if !found {
return errors.Errorf("no version found for table(%s) partition(%d)", qtid, partitionNum)
}
return c.writeLogWriter.CreateTableKeys(ctx, qtid, partitionNum, version, translations)
return c.appendTableKeysWriteLog(ctx, qtid, partitionNum, translations)
})
}
@ -993,9 +1013,9 @@ type TransactionMessage struct {
Action string
}
func intInPartitions(i int, s dax.VersionedPartitions) bool {
func intInPartitions(i int, s dax.PartitionNums) bool {
for _, a := range s {
if int(a.Num) == i {
if int(a) == i {
return true
}
}

5
cmd/backup_tar.go
View file

@ -2,12 +2,13 @@
package cmd
import (
"io"
"github.com/featurebasedb/featurebase/v3/ctl"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/spf13/cobra"
)
func newBackupTarCommand(logdest logger.Logger) *cobra.Command {
func newBackupTarCommand(logdest io.Writer) *cobra.Command {
cmd := ctl.NewBackupTarCommand(logdest)
ccmd := &cobra.Command{
Use: "backuptar",

37
cmd/dataframe-csv-loader.go Normal file
View file

@ -0,0 +1,37 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package cmd
import (
"github.com/featurebasedb/featurebase/v3/ctl"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/spf13/cobra"
)
// newImportCommand runs the FeatureBase import subcommand for ingesting bulk data.
func newDataframeCsvLoaderCommand(logdest logger.Logger) *cobra.Command {
cmd := ctl.NewDataframeCsvLoaderCommand(logdest)
loaderCmd := &cobra.Command{
Use: "dataframe-csv-loader",
Short: "load dataframe integer and floating point values into featurebase",
Long: `
`,
RunE: usageErrorWrapper(cmd),
}
flags := loaderCmd.Flags()
flags.StringVar(&cmd.Path, "csv", "", "path to csv input file")
flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
flags.StringVar(&cmd.Index, "index", "", "Destination Index. ")
flags.IntVar(&cmd.MaxCapacity, "buffer", 0, "Maximum size of of the line buffer defaults to go bufio default ")
ctl.SetTLSConfig(
flags, "",
&cmd.TLS.CertificatePath,
&cmd.TLS.CertificateKeyPath,
&cmd.TLS.CACertPath,
&cmd.TLS.SkipVerify,
&cmd.TLS.EnableClientVerification,
)
return loaderCmd
}

3
cmd/root.go
View file

@ -93,7 +93,7 @@ at https://docs.featurebase.com/.
rc.AddCommand(newChkSumCommand(logdest))
rc.AddCommand(newBackupCommand(logdest))
rc.AddCommand(newRestoreCommand(logdest))
rc.AddCommand(newBackupTarCommand(logdest))
rc.AddCommand(newBackupTarCommand(stderr))
rc.AddCommand(newRestoreTarCommand(logdest))
rc.AddCommand(newConfigCommand(stderr))
rc.AddCommand(newExportCommand(logdest))
@ -106,6 +106,7 @@ at https://docs.featurebase.com/.
rc.AddCommand(newKeygenCommand(logdest))
rc.AddCommand(newCLICommand(logdest))
rc.AddCommand(newDAXCommand(stderr))
rc.AddCommand(newDataframeCsvLoaderCommand(logdest))
rc.SetOutput(stderr)
return rc

11
cmd/server.go
View file

@ -5,6 +5,10 @@ package cmd
import (
"io"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentracer"
"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
"github.com/featurebasedb/featurebase/v3/ctl"
"github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/tracing"
@ -12,6 +16,7 @@ import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
jaegercfg "github.com/uber/jaeger-client-go/config"
"gopkg.in/DataDog/dd-trace-go.v1/profiler"
)
@ -113,8 +118,12 @@ on the configured port.`,
}
defer closer.Close()
tracing.GlobalTracer = opentracing.NewTracer(tracer, Server.Logger())
}
} else if Server.Config.DataDog.EnableTracing { // Give preference to legacy support of jaeger
t := opentracer.New(tracer.WithServiceName(Server.Config.DataDog.Service))
defer tracer.Stop()
tracing.GlobalTracer = opentracing.NewTracer(t, Server.Logger())
}
return errors.Wrap(Server.Wait(), "waiting on Server")
},
}

40
ctl/backup_tar.go
View file

@ -21,7 +21,6 @@ import (
"github.com/featurebasedb/featurebase/v3/encoding/proto"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/vprint"
"github.com/pkg/errors"
)
@ -52,7 +51,8 @@ type BackupTarCommand struct { // nolint: maligned
client *pilosa.InternalClient
// Standard input/output
logDest logger.Logger
logwriter io.Writer
logDest logger.Logger
TLS server.TLSConfig
@ -65,9 +65,10 @@ func (cmd *BackupTarCommand) Logger() logger.Logger {
}
// NewBackupTarCommand returns a new instance of BackupCommand.
func NewBackupTarCommand(logdest logger.Logger) *BackupTarCommand {
func NewBackupTarCommand(logwriter io.Writer) *BackupTarCommand {
return &BackupTarCommand{
logDest: logdest,
logwriter: logwriter,
logDest: logger.NewStandardLogger(logwriter),
RetryPeriod: time.Minute,
HeaderTimeout: time.Second * 3,
Pprof: "localhost:0",
@ -76,18 +77,24 @@ func NewBackupTarCommand(logdest logger.Logger) *BackupTarCommand {
// Run executes the main program execution.
func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) {
logger := cmd.Logger()
close, err := startProfilingServer(cmd.Pprof, logger)
if err != nil {
return errors.Wrap(err, "starting profiling server")
}
defer close()
logdest := cmd.Logger()
// Validate arguments.
if cmd.OutputPath == "" {
return fmt.Errorf("%w: -o flag required", UsageError)
}
useStdout := cmd.OutputPath == "-"
if useStdout && cmd.logwriter == os.Stdout {
logdest = logger.NewStandardLogger(os.Stderr)
}
// This was the very first thing in the function, but since logging to stdout causes file corruption
// if the tarfile is also going to stdout, we need to check that before we can safely send anything
// to the logger.
close, err := startProfilingServer(cmd.Pprof, logdest)
if err != nil {
return errors.Wrap(err, "starting profiling server")
}
defer close()
if cmd.HeaderTimeoutStr != "" {
if dur, err := time.ParseDuration(cmd.HeaderTimeoutStr); err != nil {
@ -137,6 +144,13 @@ func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) {
var w io.Writer
if useStdout {
w = os.Stdout
// if writing tarfile to stdout, the logs can't also go there or the file ends up corrupt
// redirect to stderr and log a message there to avoid this
// commented out for testing
//if dest := logger.Logger(); dest.Writer() == os.Stdout {
// dest.SetOutput(os.Stderr)
// logger.Printf("redirected logs to stderr to avoid file corruption")
//}
} else {
f, err := os.Create(cmd.OutputPath + ".tmp")
if err != nil {
@ -171,7 +185,7 @@ func (cmd *BackupTarCommand) Run(ctx context.Context) (err error) {
// Move data file to final location.
if !useStdout {
logger.Printf("writing backup: %s", cmd.OutputPath)
logdest.Printf("writing backup: %s", cmd.OutputPath)
if err := os.Rename(cmd.OutputPath+".tmp", cmd.OutputPath); err != nil {
return err
}
@ -359,7 +373,7 @@ func (cmd *BackupTarCommand) backupTarShardDataframe(ctx context.Context, tw *ta
}
filename := filepath.Join("indexes", indexName, "dataframe", fmt.Sprintf("%04d", shard))
vprint.VV("wrting %v", filename)
logger.Printf("writing %v", filename)
var buf bytes.Buffer
if _, err := buf.ReadFrom(resp.Body); err != nil {
return fmt.Errorf("copying shard data to memory: %w", err)

51
ctl/backup_tar_test.go
View file

@ -2,13 +2,11 @@ package ctl
import (
"context"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"github.com/featurebasedb/featurebase/v3/logger"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/test"
)
@ -16,27 +14,54 @@ func TestBackupTarCommand_Run(t *testing.T) {
cluster := test.MustRunCluster(t, 1)
defer cluster.Close()
cmd := cluster.GetNode(0)
indexName := "backuptar"
cmLog := logger.NewStandardLogger(io.Discard)
cm := NewBackupTarCommand(cmLog)
// this might produce some annoying spam in tests but we need to make sure log messages to
// stdout are being redirected properly when the tarfile is also going to stdout
cm := NewBackupTarCommand(os.Stdout)
hostport := cmd.API.Node().URI.HostPort()
cm.Host = hostport
dir := t.TempDir()
cm.OutputPath = filepath.Join(dir, "backuptest.tar")
resp, err := http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader("")))
_, err := cmd.API.CreateIndex(context.Background(), indexName, pilosa.IndexOptions{Keys: true, TrackExistence: true})
if err != nil {
t.Fatalf("making http request: %v", err)
t.Fatalf("creating test index: %v", err)
}
resp.Body.Close()
resp, err = http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i/field/f", strings.NewReader("")))
_, err = cmd.API.CreateField(context.Background(), indexName, "f", pilosa.OptFieldKeys())
if err != nil {
t.Fatalf("making http request: %v", err)
t.Fatalf("creating test field: %v", err)
}
resp.Body.Close()
cm.Index = "i"
cm.Index = indexName
if err := cm.Run(context.Background()); err != nil {
t.Fatalf("BackupTarCommand Run error: %s", err)
}
oldpath := cm.OutputPath
cm.OutputPath = "-"
cfpath := filepath.Join(dir, "stdouttest.tar") //capture file
cf, err := os.Create(cfpath)
if err != nil {
t.Fatalf("opening file to compare file and stdout outputs: %v", err)
}
defer cf.Close()
// I don't like this at all but it's all i'm really finding for capturing os.Stdout
old := os.Stdout
defer func() { os.Stdout = old }()
os.Stdout = cf
if err := cm.Run(context.Background()); err != nil {
t.Fatalf("BackupTarCommand Run error: %s", err)
}
fdata, err := os.ReadFile(oldpath)
if err != nil {
t.Fatalf("unable to read from direct-to-file backup: %v", err)
}
cdata, err := os.ReadFile(cfpath)
if err != nil {
t.Fatalf("unable to read from captured stdout backup: %v", err)
}
if len(fdata) != len(cdata) {
t.Fatalf("backing up to file and to stdout produced different length results")
}
}

21
ctl/cli.go
View file

@ -10,6 +10,7 @@ import (
"os"
"path/filepath"
"strings"
"time"
"github.com/chzyer/readline"
"github.com/jedib0t/go-pretty/table"
@ -23,6 +24,7 @@ import (
)
const (
defaultHost string = "localhost"
promptBegin string = "fbsql> "
promptMid string = " -> "
terminationChar string = ";"
@ -71,7 +73,7 @@ func NewCLICommand(logdest logger.Logger) *CLICommand {
}
}
return &CLICommand{
Host: "localhost",
Host: defaultHost,
HistoryPath: historyPath,
OrganizationID: "",
@ -118,7 +120,7 @@ func (cmd *CLICommand) setupClient() error {
case featurebaseTypeCloud:
fmt.Println("Detected cloud deployment")
cmd.queryer = &fbcloud.Queryer{
Host: cmd.Host,
Host: hostPort(cmd.Host, cmd.Port),
ClientID: cmd.ClientID,
Region: cmd.Region,
@ -174,6 +176,16 @@ func (cmd *CLICommand) detectFBType() (featurebaseType, error) {
typ: featurebaseTypeStandard,
},
)
} else if strings.HasPrefix(cmd.Host, "https") {
// https suggesting we might be connecting to a cloud host
trials = append(trials,
// cloud
trial{
port: "",
health: "health",
typ: featurebaseTypeCloud,
},
)
} else {
// Try default ports just in case.
trials = append(trials,
@ -192,9 +204,12 @@ func (cmd *CLICommand) detectFBType() (featurebaseType, error) {
)
}
client := http.Client{
Timeout: 100 * time.Millisecond,
}
for _, trial := range trials {
url := hostPort(cmd.Host, trial.port) + trial.health
if resp, err := http.Get(url); err != nil {
if resp, err := client.Get(url); err != nil {
continue
} else if resp.StatusCode/100 == 2 {
cmd.Port = trial.port

391
ctl/dataframe-csv-loader.go Normal file
View file

@ -0,0 +1,391 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (
"bufio"
"context"
"crypto/tls"
"encoding/gob"
"fmt"
"io"
"math"
"net/http"
"os"
"runtime"
"strconv"
"strings"
"github.com/apache/arrow/go/v10/arrow"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/client"
"github.com/featurebasedb/featurebase/v3/idk"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/pkg/errors"
)
var (
mask = uint64(pilosa.ShardWidth - 1)
Sentinal = uint64(math.MaxUint64)
)
func init() {
gob.Register(arrow.PrimitiveTypes.Int64)
gob.Register(arrow.PrimitiveTypes.Float64)
}
// TODO(rdp): add refresh token to this as well
// NewDataframeCsvLoaderCommand
type DataframeCsvLoaderCommand struct {
tlsConfig *tls.Config
Host string
Index string
// Filepath to the csv file
Path string
// max line length of csv file
MaxCapacity int
// Host:port on which to listen for pprof.
Pprof string `json:"pprof"`
TLS idk.TLSConfig
AuthToken string `flag:"auth-token" help:"Authentication Token for FeatureBase"`
SchemaManager idk.SchemaManager `flag:"-"`
// Reusable client.
client *client.Client
index *client.Index
needTranslation bool
// Standard input/output
log logger.Logger
}
// Logger returns the command's associated Logger to maintain CommandWithTLSSupport interface compatibility
func (cmd *DataframeCsvLoaderCommand) Logger() logger.Logger {
return cmd.log
}
// NewDataframeCsvLoaderCommand returns a new instance of DataframeCsvLoaderCommand.
func NewDataframeCsvLoaderCommand(logdest logger.Logger) *DataframeCsvLoaderCommand {
return &DataframeCsvLoaderCommand{
log: logdest,
Pprof: "localhost:0",
}
}
func (cmd *DataframeCsvLoaderCommand) setupClient() (*tls.Config, error) {
var tlsConfig *tls.Config
var err error
opts := []client.ClientOption{}
if cmd.TLS.CertificatePath != "" {
tlsConfig, err = idk.GetTLSConfig(&cmd.TLS, cmd.log)
if err != nil {
return nil, errors.Wrap(err, "getting TLS config")
}
opts = append(opts, client.OptClientTLSConfig(tlsConfig))
} else {
opts = append(opts,
client.OptClientRetries(2),
client.OptClientTotalPoolSize(1000),
client.OptClientPoolSizePerRoute(400),
)
}
cmd.client, err = client.NewClient([]string{cmd.Host}, opts...)
if err != nil {
return nil, err
}
cmd.client.AuthToken = cmd.AuthToken
if err != nil {
return nil, errors.Wrap(err, "getting featurebase client")
}
return tlsConfig, nil
}
func (cmd *DataframeCsvLoaderCommand) Setup() (err error) {
// setup logging
cmd.log = logger.NewStandardLogger(os.Stderr)
if cmd.Pprof != "" {
go func() {
runtime.SetBlockProfileRate(10000000) // 1 sample per 10 ms
runtime.SetMutexProfileFraction(100) // 1% sampling
cmd.log.Printf("Listening for /debug/pprof/ and /debug/fgprof on '%s'", cmd.Pprof)
cmd.log.Printf("%v", http.ListenAndServe(cmd.Pprof, nil))
}()
}
// set up Pilosa client
_, err = cmd.setupClient()
if err != nil {
return errors.Wrap(err, "setting up client")
}
if cmd.AuthToken != "" {
cmd.AuthToken = "Bearer " + cmd.AuthToken // Gets added to context
}
return nil
}
// Run executes the dataload.
func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) {
err = cmd.Setup()
if err != nil {
return err
}
logger := cmd.Logger()
close, err := startProfilingServer(cmd.Pprof, logger)
if err != nil {
return errors.Wrap(err, "starting profiling server")
}
defer close()
// Validate arguments.
if cmd.Path == "" {
return fmt.Errorf("%w: --csv flag required", UsageError)
}
readFile, err := os.Open(cmd.Path)
if err != nil {
return err
}
fields := make([]arrow.Field, 0)
fields = append(fields, arrow.Field{Name: "_ID", Type: arrow.PrimitiveTypes.Int64})
fileScanner := bufio.NewScanner(readFile)
fileScanner.Split(bufio.ScanLines)
// need for really long csv lines
var buf []byte
if cmd.MaxCapacity > 0 {
buf = make([]byte, cmd.MaxCapacity)
fileScanner.Buffer(buf, cmd.MaxCapacity)
}
total := 0
if fileScanner.Scan() {
total++
t := fileScanner.Text()
p := strings.Split(t, ",")
for _, col := range p[1:] {
col = strings.TrimSpace(col)
cmd.Logger().Infof("checking %v", col)
name := col[:strings.LastIndex(col, "__")]
cmd.Logger().Infof("name:%v", name)
if strings.HasSuffix(col, "__I") {
fields = append(fields, arrow.Field{Name: name, Type: arrow.PrimitiveTypes.Int64})
} else if strings.HasSuffix(col, "__F") {
fields = append(fields, arrow.Field{Name: name, Type: arrow.PrimitiveTypes.Float64})
} else {
return errors.New("invalid format for type")
}
}
} else {
return errors.Wrap(fileScanner.Err(), "No header")
}
schema, err := cmd.client.Schema()
if err != nil {
return err
}
idx := schema.Index(cmd.Index)
if idx.Opts().Keys() {
cmd.needTranslation = true
cmd.index = idx
}
arrowSchema := arrow.NewSchema(fields, nil)
keys := make([]string, 0)
lookup := make(map[string]uint64)
if cmd.needTranslation {
for fileScanner.Scan() {
t := fileScanner.Text()
r := t[:strings.Index(t, ",")]
_, ok := lookup[r]
if !ok {
keys = append(keys, r)
lookup[r] = Sentinal
}
}
cmd.Logger().Infof("Translate Keys %d", total)
ids, err := cmd.client.CreateIndexKeys(cmd.index, keys...)
if err != nil {
return err
}
lookup = ids
}
sharder := &Sharder{
shards: make(map[uint64]*ShardDiff),
schema: arrowSchema,
index: cmd.Index,
log: cmd.log,
}
readFile.Seek(0, io.SeekStart)
fileScanner = bufio.NewScanner(readFile)
if cmd.MaxCapacity > 0 {
fileScanner.Buffer(buf, cmd.MaxCapacity)
}
fileScanner.Split(bufio.ScanLines)
fileScanner.Scan() // skip the header
cmd.Logger().Infof("Build the dataframe input package in memory")
id := uint64(0)
for fileScanner.Scan() {
records := strings.Split(fileScanner.Text(), ",")
if cmd.needTranslation {
id = lookup[records[0]]
} else {
id, err = strconv.ParseUint(records[0], 10, 64)
if err != nil {
return err
}
}
shard := id / pilosa.ShardWidth
shardFile, err := sharder.GetShard(shard)
if err != nil {
return err
}
shardRow := int64(id & mask)
shardFile.SetRow(shardRow)
for i, rec := range records {
if i == 0 {
shardFile.SetIntValue(i, shardRow, int64(id))
} else {
rec = strings.TrimSpace(rec)
switch arrowSchema.Field(i).Type {
case arrow.PrimitiveTypes.Int64:
val, err := strconv.ParseInt(rec, 10, 64)
if err != nil {
shardFile.SetIntValue(i, shardRow, 0)
continue
}
shardFile.SetIntValue(i, shardRow, val)
case arrow.PrimitiveTypes.Float64:
val, err := strconv.ParseFloat(rec, 64)
if err != nil {
shardFile.SetFloatValue(i, shardRow, 0)
continue
}
shardFile.SetFloatValue(i, shardRow, val)
default:
return errors.New("unhandled arrow type type")
}
}
}
}
cmd.Logger().Infof("sending package to featurebase")
err = sharder.Store(arrowSchema, cmd.client)
if err != nil {
return err
}
return err
}
type pair struct {
col int
row uint64
}
type ShardDiff struct {
columns []interface{}
rows []int64
null map[pair]struct{}
shard uint64
// Standard input/output
log logger.Logger
}
func NewShardDiff(shard uint64, log logger.Logger) (*ShardDiff, error) {
return &ShardDiff{shard: shard, log: log}, nil
}
type Number interface {
int64 | float64
}
func (s *ShardDiff) SetIntValue(col int, row int64, val int64) {
slice := s.columns[col].([]int64)
s.columns[col] = append(slice, val)
}
func (s *ShardDiff) SetFloatValue(col int, row int64, val float64) {
slice := s.columns[col].([]float64)
s.columns[col] = append(slice, val)
}
func (s *ShardDiff) SetNulll(col int, row uint64) {
s.null[pair{col: col, row: row}] = struct{}{}
}
func (s *ShardDiff) SetRow(row int64) {
s.rows = append(s.rows, row)
}
func (s *ShardDiff) Setup(schema *arrow.Schema) {
for _, f := range schema.Fields() {
switch f.Type {
case arrow.PrimitiveTypes.Int64:
s.columns = append(s.columns, make([]int64, 0))
case arrow.PrimitiveTypes.Float64:
s.columns = append(s.columns, make([]float64, 0))
}
}
}
func makeSimpleSchema(a *arrow.Schema) []pilosa.NameType {
nt := make([]pilosa.NameType, len(a.Fields()))
for i := 0; i < len(a.Fields()); i++ {
f := a.Field(i)
nt[i] = pilosa.NameType{Name: f.Name, DataType: f.Type}
}
return nt
}
func (s *ShardDiff) Store(index string, schema *arrow.Schema, fb *client.Client) error {
s.log.Infof("dataframe for shard %v:%v:", index, s.shard)
request := &pilosa.ChangesetRequest{}
request.Columns = s.columns
request.ShardIds = s.rows
request.SimpleSchema = makeSimpleSchema(schema)
_, err := fb.ApplyDataframeChangeset(index, request, s.shard)
return err
}
func (s *ShardDiff) IsValid(col int, row int) bool {
_, ok := s.null[pair{col: col, row: uint64(row)}]
return !ok
}
type Sharder struct {
shards map[uint64]*ShardDiff
index string
schema *arrow.Schema
// Standard input/output
log logger.Logger
}
func (s *Sharder) GetShard(shard uint64) (*ShardDiff, error) {
f, ok := s.shards[shard]
if ok {
return f, nil
}
f, err := NewShardDiff(shard, s.log)
f.Setup(s.schema)
if err != nil {
return nil, err
}
s.shards[shard] = f
return f, nil
}
func (s *Sharder) Store(schema *arrow.Schema, client *client.Client) error {
for _, f := range s.shards {
err := f.Store(s.index, schema, client)
if err != nil {
return err
}
}
return nil
}

74
ctl/dataframe-csv-loader_test.go Normal file
View file

@ -0,0 +1,74 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package ctl
import (
"context"
"io"
"testing"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/test"
"github.com/featurebasedb/featurebase/v3/testhook"
)
func TestDataframeCsvLoaderCommand(t *testing.T) {
cluster := test.MustRunCluster(t, 1, []server.CommandOption{server.OptCommandServerOptions(pilosa.OptServerIsDataframeEnabled(true))})
defer cluster.Close()
cmd := cluster.GetNode(0)
t.Run("basic", func(t *testing.T) {
cmLog := logger.NewStandardLogger(io.Discard)
cm := NewDataframeCsvLoaderCommand(cmLog)
file, err := testhook.TempFile(t, "import.csv")
if err != nil {
t.Fatalf("creating tempfile: %v", err)
}
_, err = file.Write([]byte("id,val__I\n1,2\n3,4\n5,6"))
if err != nil {
t.Fatalf("writing to tempfile: %v", err)
}
ctx := context.Background()
if err != nil {
t.Fatal(err)
}
index := "non-keyed"
cmd.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: false})
cm.Host = cmd.API.Node().URI.HostPort()
cm.Path = file.Name()
cm.Index = index
err = cm.Run(ctx)
if err != nil {
t.Fatalf("DataframeCsvLoader Run doesn't work: %s", err)
}
})
t.Run("keyed", func(t *testing.T) {
cmLog := logger.NewStandardLogger(io.Discard)
cm := NewDataframeCsvLoaderCommand(cmLog)
file, err := testhook.TempFile(t, "import_key.csv")
if err != nil {
t.Fatalf("creating tempfile: %v", err)
}
_, err = file.Write([]byte("id,val__I\nA,2\nB,4\nC,6"))
if err != nil {
t.Fatalf("writing to tempfile: %v", err)
}
ctx := context.Background()
if err != nil {
t.Fatal(err)
}
index := "keyed"
cmd.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true})
cm.Host = cmd.API.Node().URI.HostPort()
cm.Path = file.Name()
cm.Index = index
err = cm.Run(ctx)
if err != nil {
t.Fatalf("DataframeCsvLoader Run doesn't work: %s", err)
}
})
}

1
ctl/dax.go
View file

@ -18,6 +18,7 @@ func BuildDAXFlags(cmd *cobra.Command, srv *server.Command) {
flags.BoolVar(&srv.Config.MDS.Run, "mds.run", srv.Config.MDS.Run, "Run the MDS service in process.")
flags.DurationVar(&srv.Config.MDS.Config.RegistrationBatchTimeout, "mds.config.registration-batch-timeout", srv.Config.MDS.Config.RegistrationBatchTimeout, "Timeout for node registration batches.")
flags.StringVar(&srv.Config.MDS.Config.DataDir, "mds.config.data-dir", srv.Config.MDS.Config.DataDir, "MDS directory to use in process.")
flags.DurationVar(&srv.Config.MDS.Config.SnappingTurtleTimeout, "mds.config.snapping-turtle-timeout", srv.Config.MDS.Config.SnappingTurtleTimeout, "Period for running automatic snapshotting routine.")
// WriteLogger
flags.BoolVar(&srv.Config.WriteLogger.Run, "writelogger.run", srv.Config.WriteLogger.Run, "Run the WriteLogger service in process.")

4
ctl/server.go
View file

@ -133,10 +133,13 @@ func serverFlagSet(srv *server.Config, prefix string) *pflag.FlagSet {
flags.StringSliceVar(&srv.Auth.ConfiguredIPs, pre("auth.configured-ips"), srv.Auth.ConfiguredIPs, "List of configured IPs allowed for ingest")
flags.BoolVar(&srv.DataDog.Enable, pre("datadog.enable"), false, "enable continuous profiling with DataDog cloud service, Note you must have DataDog agent installed")
flags.BoolVar(&srv.DataDog.EnableTracing, pre("datadog.enable-tracing"), false, "Enable continuous tracing with DataDog cloud service, this flag is mutually exclusive to tracing.* parameters")
flags.StringVar(&srv.DataDog.Service, pre("datadog.service"), "default-service", "The Datadog service name, for example my-web-app")
flags.StringVar(&srv.DataDog.Env, pre("datadog.env"), "default-env", "The Datadog environment name, for example, production")
flags.StringVar(&srv.DataDog.Version, pre("datadog.version"), "default-version", "The version of your application")
flags.StringVar(&srv.DataDog.Tags, pre("datadog.tags"), "molecula", "The tags to apply to an uploaded profile. Must be a list of in the format <KEY1>:<VALUE1>,<KEY2>:<VALUE2>")
flags.BoolVar(&srv.DataDog.CPUProfile, pre("datadog.cpu-profile"), true, "golang pprof cpu profile ")
flags.BoolVar(&srv.DataDog.HeapProfile, pre("datadog.heap-profile"), true, "golang pprof heap profile")
flags.BoolVar(&srv.DataDog.MutexProfile, pre("datadog.mutex-profile"), false, "golang pprof mutex profile")
@ -144,6 +147,7 @@ func serverFlagSet(srv *server.Config, prefix string) *pflag.FlagSet {
flags.BoolVar(&srv.DataDog.BlockProfile, pre("datadog.block-profile"), false, "golang pprof goroutine ")
flags.BoolVar(&srv.Dataframe.Enable, pre("dataframe.enable"), false, "EXPERIMENTAL enable support for Apply and Arrow")
flags.BoolVar(&srv.Dataframe.UseParquet, pre("dataframe.use-parquet"), false, "EXPERIMENTAL use parquet for file format")
return flags
}

5
dax/Makefile
View file

@ -78,11 +78,14 @@ dc-logs-%:
dc-prereqs:
mkdir -p ../.quick
dc-cli:
featurebase cli --host localhost --port 8080 --org-id=testorg --db-id=testdb
# This is just an example. For it to work, you'll first need to:
# featurebase cli --host localhost --port 8080 --org-id=testorg --db-id=testdb
# create table keysidstbl2 (_id string, slice idset);
dc-datagen:
docker-compose run datagen --end-at=500 --pilosa.batch-size=500 --featurebase.table-name=keysidstbl2
docker-compose run datagen --end-at=500 --pilosa.batch-size=500 --featurebase.table-name=keysidstbl2
dc-exec-%:
docker-compose exec $* /bin/sh

768
dax/boltdb/versionstore.go
View file

@ -1,768 +0,0 @@
package boltdb
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"strconv"
"strings"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/inmem"
"github.com/featurebasedb/featurebase/v3/errors"
"github.com/featurebasedb/featurebase/v3/logger"
)
var (
bucketTables = Bucket("versionStoreTables")
bucketShards = Bucket("versionStoreShards")
bucketTableKeys = Bucket("versionStoreTableKeys")
bucketFieldKeys = Bucket("versionStoreFieldKeys")
)
// VersionStoreBuckets defines the buckets used by this package. It can be
// called during setup to create the buckets ahead of time.
var VersionStoreBuckets []Bucket = []Bucket{
bucketTables,
bucketShards,
bucketTableKeys,
bucketFieldKeys,
}
// Ensure type implements interface.
var _ dax.VersionStore = (*VersionStore)(nil)
// VersionStore manages all version info for shard, table keys, and field keys.
type VersionStore struct {
db *DB
logger logger.Logger
}
// NewVersionStore returns a new instance of VersionStore with default values.
func NewVersionStore(db *DB, logger logger.Logger) *VersionStore {
return &VersionStore{
db: db,
logger: logger,
}
}
func (s *VersionStore) AddTable(ctx context.Context, qtid dax.QualifiedTableID) error {
tx, err := s.db.BeginTx(ctx, true)
if err != nil {
return errors.Wrap(err, "getting transaction")
}
defer tx.Rollback()
bkt := tx.Bucket(bucketTables)
if bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketTables)
}
if val := bkt.Get(tableKey(qtid)); val != nil {
return dax.NewErrTableIDExists(qtid)
}
// The assumption is that we may store information about the table (other
// than just the fact that it exists). So for now, the value is an empty
// JSON object.
val := []byte("{}")
if err := bkt.Put(tableKey(qtid), val); err != nil {
return errors.Wrap(err, "putting table")
}
// Add the table to the "table index" of the other buckets.
//
// Shards
if bkt := tx.Bucket(bucketShards); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketShards)
} else if err := bkt.Put(tableKey(qtid), val); err != nil {
return errors.Wrap(err, "putting table into shards")
}
// TableKeys.
if bkt := tx.Bucket(bucketTableKeys); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketTableKeys)
} else if err := bkt.Put(tableKey(qtid), val); err != nil {
return errors.Wrap(err, "putting table into table keys")
}
// FieldKeys.
if bkt := tx.Bucket(bucketFieldKeys); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketFieldKeys)
} else if err := bkt.Put(tableKey(qtid), val); err != nil {
return errors.Wrap(err, "putting table into field keys")
}
return tx.Commit()
}
func (s *VersionStore) RemoveTable(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedShards, dax.VersionedPartitions, error) {
tx, err := s.db.BeginTx(ctx, true)
if err != nil {
return nil, nil, err
}
defer tx.Rollback()
// Get the shards and partitions before deleting by table.
shards, err := s.getShards(ctx, tx, qtid)
if err != nil {
return nil, nil, err
}
partitions, err := s.getPartitions(ctx, tx, qtid)
if err != nil {
return nil, nil, err
}
if err := removeTable(ctx, tx, qtid); err != nil {
return nil, nil, err
}
if err := tx.Commit(); err != nil {
return nil, nil, err
}
return shards, partitions, nil
}
func removeTable(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID) error {
// Tables.
if bkt := tx.Bucket(bucketTables); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketTables)
} else if err := bkt.Delete(tableKey(qtid)); err != nil {
return errors.Wrap(err, "deleting table")
}
// Shards.
if bkt := tx.Bucket(bucketShards); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketShards)
} else if err := bkt.Delete(tableKey(qtid)); err != nil {
return errors.Wrap(err, "deleting table in shards")
} else if err := deleteByPrefix(tx, bucketShards, []byte(fmt.Sprintf(prefixFmtShards, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))); err != nil {
return errors.Wrap(err, "deleting shards for table")
}
// TableKeys.
if bkt := tx.Bucket(bucketTableKeys); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketTableKeys)
} else if err := bkt.Delete(tableKey(qtid)); err != nil {
return errors.Wrap(err, "deleting table in table keys")
} else if err := deleteByPrefix(tx, bucketTableKeys, []byte(fmt.Sprintf(prefixFmtTableKeys, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))); err != nil {
return errors.Wrap(err, "deleting table keys for table")
}
// FieldKeys.
if bkt := tx.Bucket(bucketFieldKeys); bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketFieldKeys)
} else if err := bkt.Delete(tableKey(qtid)); err != nil {
return errors.Wrap(err, "deleting table in field keys")
} else if err := deleteByPrefix(tx, bucketFieldKeys, []byte(fmt.Sprintf(prefixFmtFieldKeys, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))); err != nil {
return errors.Wrap(err, "deleting field keys for table")
}
return nil
}
func deleteByPrefix(tx *Tx, bucket Bucket, prefix []byte) error {
bkt := tx.Bucket(bucket)
cursor := bkt.Cursor()
// Deleting keys within the for loop seems to cause Next() to skip the next
// matching key because the Delete() call pops the item and effectively
// moves the cursor forward. Then calling Next() skips the item that was
// being pointed to after the delete. So, we're going to make a list of keys
// to delete, and then delete them outside of the cursor logic.
var keysToDelete [][]byte
for k, _ := cursor.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = cursor.Next() {
keysToDelete = append(keysToDelete, k)
}
for _, k := range keysToDelete {
if err := bkt.Delete(k); err != nil {
return errors.Wrapf(err, "deleting key: %s", k)
}
}
return nil
}
func (s *VersionStore) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.VersionedShard) error {
tx, err := s.db.BeginTx(ctx, true)
if err != nil {
return errors.Wrap(err, "getting transaction")
}
defer tx.Rollback()
for _, shard := range shards {
if err := createShard(ctx, tx, qtid, shard); err != nil {
return errors.Wrap(err, "creating shard")
}
}
return tx.Commit()
}
func createShard(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, shard dax.VersionedShard) error {
// TODO: validate data more formally
if shard.Version < 0 {
return errors.New(errors.ErrUncoded, fmt.Sprintf("invalid shard version: %d", shard.Version))
}
bkt := tx.Bucket(bucketShards)
if bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketShards)
}
// Ensure the table exists.
if val := bkt.Get(tableKey(qtid)); val == nil {
return dax.NewErrTableIDDoesNotExist(qtid)
}
vsn := make([]byte, 8)
binary.LittleEndian.PutUint64(vsn, uint64(shard.Version))
return bkt.Put(shardKey(qtid, shard.Num), vsn)
}
func (s *VersionStore) Shards(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedShards, bool, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, false, errors.Wrap(err, "getting tx")
}
defer tx.Rollback()
shards, err := s.getShards(ctx, tx, qtid)
if err != nil {
return nil, false, errors.Wrap(err, "getting shards")
}
return shards, true, nil
}
func (s *VersionStore) getShards(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID) (dax.VersionedShards, error) {
c := tx.Bucket(bucketShards).Cursor()
// Deserialize rows into Shard objects.
shards := make(dax.VersionedShards, 0)
prefix := []byte(fmt.Sprintf(prefixFmtShards, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
if v == nil {
s.logger.Printf("nil value for key: %s", k)
continue
}
var shard dax.VersionedShard
shardNum, err := keyShardNum(k)
if err != nil {
return nil, errors.Wrapf(err, "getting shardNum from key: %v", k)
}
shard.Num = shardNum
shard.Version = int(binary.LittleEndian.Uint64(v))
shards = append(shards, shard)
}
return shards, nil
}
// ShardVersion return the current version for the given table/shardNum.
// If a version is not being tracked, it returns a bool value of false.
func (s *VersionStore) ShardVersion(ctx context.Context, qtid dax.QualifiedTableID, shardNum dax.ShardNum) (int, bool, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return -1, false, err
}
defer tx.Rollback()
return getShardVersion(ctx, tx, qtid, shardNum)
}
func getShardVersion(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, shardNum dax.ShardNum) (int, bool, error) {
version := -1
bkt := tx.Bucket(bucketShards)
if bkt == nil {
return version, false, errors.Errorf(ErrFmtBucketNotFound, bucketShards)
}
b := bkt.Get(shardKey(qtid, shardNum))
if b == nil {
return version, false, nil
}
version = int(binary.LittleEndian.Uint64(b))
return version, true, nil
}
func (s *VersionStore) ShardTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, errors.Wrap(err, "beginning tx")
}
defer tx.Rollback()
return s.getTableIDs(ctx, tx, qual, bucketShards)
}
func (s *VersionStore) getTableIDs(ctx context.Context, tx *Tx, qual dax.TableQualifier, bucket Bucket) (dax.TableIDs, error) {
c := tx.Bucket(bucket).Cursor()
// Deserialize rows into Tables objects.
tableIDs := make(dax.TableIDs, 0)
prefix := []byte(fmt.Sprintf(prefixFmtTables, qual.OrganizationID, qual.DatabaseID))
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
if v == nil {
s.logger.Printf("nil value for key: %s", k)
continue
}
var tableID dax.TableID
tableID, err := keyTableID(k)
if err != nil {
return nil, errors.Wrapf(err, "getting table name from key: %v", k)
}
tableIDs = append(tableIDs, tableID)
}
return tableIDs, nil
}
func (s *VersionStore) bucketTables(ctx context.Context, bucket Bucket) ([]dax.QualifiedTableID, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, errors.Wrap(err, "beginning tx")
}
defer tx.Rollback()
c := tx.Bucket(bucket).Cursor()
// Deserialize rows into Tables objects.
qtids := make([]dax.QualifiedTableID, 0)
prefix := []byte(prefixTables)
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
if v == nil {
s.logger.Printf("nil value for key: %s", k)
continue
}
qtid, err := keyQualifiedTableID(k)
if err != nil {
return nil, errors.Wrapf(err, "getting qualified table id from key: %v", k)
}
qtids = append(qtids, qtid)
}
return qtids, nil
}
// AddPartitions adds new partitions to be managed by VersionStore. It returns
// the number of partitions added or an error.
func (s *VersionStore) AddPartitions(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.VersionedPartition) error {
tx, err := s.db.BeginTx(ctx, true)
if err != nil {
return errors.Wrap(err, "getting transaction")
}
defer tx.Rollback()
for _, partition := range partitions {
if err := createPartition(ctx, tx, qtid, partition); err != nil {
return errors.Wrap(err, "creating partition")
}
}
return tx.Commit()
}
func createPartition(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, partition dax.VersionedPartition) error {
// TODO: validate data more formally
if partition.Version < 0 {
return errors.New(errors.ErrUncoded, fmt.Sprintf("invalid partition version: %d", partition.Version))
}
bkt := tx.Bucket(bucketTableKeys)
if bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketTableKeys)
}
// Ensure the table exists.
if val := bkt.Get(tableKey(qtid)); val == nil {
return dax.NewErrTableIDDoesNotExist(qtid)
}
vsn := make([]byte, 8)
binary.LittleEndian.PutUint64(vsn, uint64(partition.Version))
return bkt.Put(partitionKey(qtid, partition.Num), vsn)
}
func (s *VersionStore) Partitions(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedPartitions, bool, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, false, errors.Wrap(err, "getting tx")
}
defer tx.Rollback()
partitions, err := s.getPartitions(ctx, tx, qtid)
if err != nil {
return nil, false, errors.Wrap(err, "getting partitions")
}
return partitions, true, nil
}
func (s *VersionStore) getPartitions(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID) (dax.VersionedPartitions, error) {
c := tx.Bucket(bucketTableKeys).Cursor()
// Deserialize rows into Partition objects.
partitions := make(dax.VersionedPartitions, 0)
prefix := []byte(fmt.Sprintf(prefixFmtTableKeys, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
if v == nil {
s.logger.Printf("nil value for key: %s", k)
continue
}
var partition dax.VersionedPartition
partitionNum, err := keyPartitionNum(k)
if err != nil {
return nil, errors.Wrapf(err, "getting partitionNum from key: %v", k)
}
partition.Num = partitionNum
partition.Version = int(binary.LittleEndian.Uint64(v))
partitions = append(partitions, partition)
}
return partitions, nil
}
func (s *VersionStore) PartitionVersion(ctx context.Context, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) (int, bool, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return -1, false, err
}
defer tx.Rollback()
return getPartitionVersion(ctx, tx, qtid, partitionNum)
}
func getPartitionVersion(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) (int, bool, error) {
version := -1
bkt := tx.Bucket(bucketTableKeys)
if bkt == nil {
return version, false, errors.Errorf(ErrFmtBucketNotFound, bucketTableKeys)
}
b := bkt.Get(partitionKey(qtid, partitionNum))
if b == nil {
return version, false, nil
}
version = int(binary.LittleEndian.Uint64(b))
return version, true, nil
}
func (s *VersionStore) PartitionTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, errors.Wrap(err, "beginning tx")
}
defer tx.Rollback()
return s.getTableIDs(ctx, tx, qual, bucketTableKeys)
}
// AddFields adds new fields to be managed by VersionStore. It returns the
// number of fields added or an error.
func (s *VersionStore) AddFields(ctx context.Context, qtid dax.QualifiedTableID, fields ...dax.VersionedField) error {
tx, err := s.db.BeginTx(ctx, true)
if err != nil {
return err
}
defer tx.Rollback()
for _, field := range fields {
if err := createFieldVersion(ctx, tx, qtid, field); err != nil {
return errors.Wrap(err, "creating field version")
}
}
return tx.Commit()
}
func createFieldVersion(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, field dax.VersionedField) error {
// TODO: validate data more formally
if field.Version < 0 {
return errors.New(errors.ErrUncoded, fmt.Sprintf("invalid field version: %d", field.Version))
}
bkt := tx.Bucket(bucketFieldKeys)
if bkt == nil {
return errors.Errorf(ErrFmtBucketNotFound, bucketFieldKeys)
}
// Ensure the table exists.
if val := bkt.Get(tableKey(qtid)); val == nil {
return dax.NewErrTableIDDoesNotExist(qtid)
}
vsn := make([]byte, 8)
binary.LittleEndian.PutUint64(vsn, uint64(field.Version))
return bkt.Put(fieldKey(qtid, field.Name), vsn)
}
func (s *VersionStore) Fields(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedFields, bool, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, false, errors.Wrap(err, "getting tx")
}
defer tx.Rollback()
fields, err := s.getFields(ctx, tx, qtid)
if err != nil {
return nil, false, errors.Wrap(err, "getting fields")
}
return fields, true, nil
}
func (s *VersionStore) getFields(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID) (dax.VersionedFields, error) {
c := tx.Bucket(bucketFieldKeys).Cursor()
// Deserialize rows into FieldVersion objects.
fieldVersions := make(dax.VersionedFields, 0)
prefix := []byte(fmt.Sprintf(prefixFmtFieldKeys, qtid.OrganizationID, qtid.DatabaseID, qtid.ID))
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
if v == nil {
s.logger.Printf("nil value for key: %s", k)
continue
}
var fieldVersion dax.VersionedField
fieldName, err := keyFieldName(k)
if err != nil {
return nil, errors.Wrapf(err, "getting partitionNum from key: %v", k)
}
fieldVersion.Name = fieldName
fieldVersion.Version = int(binary.LittleEndian.Uint64(v))
fieldVersions = append(fieldVersions, fieldVersion)
}
return fieldVersions, nil
}
func (s *VersionStore) FieldVersion(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName) (int, bool, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return -1, false, err
}
defer tx.Rollback()
return getFieldVersion(ctx, tx, qtid, field)
}
func getFieldVersion(ctx context.Context, tx *Tx, qtid dax.QualifiedTableID, field dax.FieldName) (int, bool, error) {
version := -1
bkt := tx.Bucket(bucketFieldKeys)
if bkt == nil {
return version, false, errors.Errorf(ErrFmtBucketNotFound, bucketFieldKeys)
}
b := bkt.Get(fieldKey(qtid, field))
if b == nil {
return version, false, nil
}
version = int(binary.LittleEndian.Uint64(b))
return version, true, nil
}
func (s *VersionStore) FieldTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
return nil, errors.Wrap(err, "beginning tx")
}
defer tx.Rollback()
return s.getTableIDs(ctx, tx, qual, bucketFieldKeys)
}
// Copy returns an in-memory copy of VersionStore.
func (s *VersionStore) Copy(ctx context.Context) (dax.VersionStore, error) {
new := inmem.NewVersionStore()
// shards.
qtids, err := s.bucketTables(ctx, bucketShards)
if err != nil {
return nil, errors.Wrap(err, "getting shard tables")
}
for _, qtid := range qtids {
shards, found, err := s.Shards(ctx, qtid)
if err != nil {
return nil, errors.Wrap(err, "getting shards")
} else if !found {
continue
}
_ = new.AddTable(ctx, qtid)
new.AddShards(ctx, qtid, shards...)
}
// tableKeys.
qtids, err = s.bucketTables(ctx, bucketTableKeys)
if err != nil {
return nil, errors.Wrap(err, "getting table key tables")
}
for _, qtid := range qtids {
partitions, found, err := s.Partitions(ctx, qtid)
if err != nil {
return nil, errors.Wrap(err, "getting partitions")
} else if !found {
continue
}
_ = new.AddTable(ctx, qtid)
new.AddPartitions(ctx, qtid, partitions...)
}
// fieldKeys.
qtids, err = s.bucketTables(ctx, bucketFieldKeys)
if err != nil {
return nil, errors.Wrap(err, "getting field key tables")
}
for _, qtid := range qtids {
fields, found, err := s.Fields(ctx, qtid)
if err != nil {
return nil, errors.Wrap(err, "getting fields")
} else if !found {
continue
}
_ = new.AddTable(ctx, qtid)
new.AddFields(ctx, qtid, fields...)
}
return new, nil
}
/////////////////////////////////////////////////////////
const (
prefixShards = "shards/"
prefixFmtShards = prefixShards + "%s/%s/%s/"
prefixTableKeys = "tablekeys/"
prefixFmtTableKeys = prefixTableKeys + "%s/%s/%s/"
prefixFieldKeys = "fieldkeys/"
prefixFmtFieldKeys = prefixFieldKeys + "%s/%s/%s/"
prefixTables = "tables/"
prefixFmtTables = prefixTables + "%s/%s/"
)
// tableKey returns a key based on table name.
func tableKey(qtid dax.QualifiedTableID) []byte {
qual := qtid.TableQualifier
key := fmt.Sprintf(prefixFmtTables+"%s", qual.OrganizationID, qual.DatabaseID, qtid.ID)
return []byte(key)
}
// keyTableID gets the table ID out of the key.
func keyTableID(key []byte) (dax.TableID, error) {
parts := strings.Split(string(key), "/")
if len(parts) != 4 {
return "", errors.New(errors.ErrUncoded, "table key format expected: `tables/orgID/dbID/tableID`")
}
return dax.TableID(parts[3]), nil
}
// keyQualifiedTableID gets the qualified table ID out of the key.
func keyQualifiedTableID(key []byte) (dax.QualifiedTableID, error) {
parts := strings.Split(string(key), "/")
if len(parts) != 4 {
return dax.QualifiedTableID{}, errors.New(errors.ErrUncoded, "table key format expected: `tables/orgID/dbID/tableID`")
}
return dax.NewQualifiedTableID(
dax.NewTableQualifier(dax.OrganizationID(parts[1]), dax.DatabaseID(parts[2])),
dax.TableID(parts[3]),
), nil
}
// shardKey returns a key based on table and shard.
func shardKey(qtid dax.QualifiedTableID, shard dax.ShardNum) []byte {
key := fmt.Sprintf(prefixFmtShards+"%d", qtid.OrganizationID, qtid.DatabaseID, qtid.ID, shard)
return []byte(key)
}
// keyShardNum gets the shardNum out of the key.
func keyShardNum(key []byte) (dax.ShardNum, error) {
parts := strings.Split(string(key), "/")
if len(parts) != 5 {
return 0, errors.New(errors.ErrUncoded, "shard key format expected: `shards/orgID/dbID/table/shard`")
}
intVar, err := strconv.Atoi(parts[4])
if err != nil {
return 0, errors.Wrapf(err, "converting string to shardNum: %s", parts[4])
}
return dax.ShardNum(intVar), nil
}
// partitionKey returns a key based on table and partition.
func partitionKey(qtid dax.QualifiedTableID, partition dax.PartitionNum) []byte {
key := fmt.Sprintf(prefixFmtTableKeys+"%d", qtid.OrganizationID, qtid.DatabaseID, qtid.ID, partition)
return []byte(key)
}
// keyPartitionNum gets the partitionNum out of the key.
func keyPartitionNum(key []byte) (dax.PartitionNum, error) {
parts := strings.Split(string(key), "/")
if len(parts) != 5 {
return 0, errors.New(errors.ErrUncoded, "partition key format expected: `tablekeys/orgID/dbID/table/partition`")
}
intVar, err := strconv.Atoi(parts[4])
if err != nil {
return 0, errors.Wrapf(err, "converting string to partitionNum: %s", parts[4])
}
return dax.PartitionNum(intVar), nil
}
// fieldKey returns a key based on table and field.
func fieldKey(qtid dax.QualifiedTableID, field dax.FieldName) []byte {
key := fmt.Sprintf(prefixFmtFieldKeys+"%s", qtid.OrganizationID, qtid.DatabaseID, qtid.ID, field)
return []byte(key)
}
// keyFieldName gets the fieldName out of the key.
func keyFieldName(key []byte) (dax.FieldName, error) {
parts := strings.Split(string(key), "/")
if len(parts) != 5 {
return "", errors.New(errors.ErrUncoded, "field key format expected: `fieldkeys/orgID/dbID/table/field`")
}
return dax.FieldName(parts[4]), nil
}

388
dax/boltdb/versionstore_test.go
View file

@ -1,388 +0,0 @@
package boltdb_test
import (
"context"
"fmt"
"sort"
"testing"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/boltdb"
testbolt "github.com/featurebasedb/featurebase/v3/dax/test/boltdb"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/stretchr/testify/assert"
)
func TestVersionStore(t *testing.T) {
db := testbolt.MustOpenDB(t)
defer testbolt.MustCloseDB(t, db)
ctx := context.Background()
t.Cleanup(func() {
testbolt.CleanupDB(t, db.Path())
})
orgID := dax.OrganizationID("acme")
dbID := dax.DatabaseID("db1")
qual := dax.NewTableQualifier(orgID, dbID)
// Initialize the buckets.
assert.NoError(t, db.InitializeBuckets(boltdb.VersionStoreBuckets...))
t.Run("Tables", func(t *testing.T) {
vs := boltdb.NewVersionStore(db, logger.NopLogger)
qtids := newQualifiedTableIDs(t, qual, 3)
qtid1 := qtids[0]
qtid2 := qtids[1]
qtid3 := qtids[2]
defer vs.RemoveTable(ctx, qtid1)
defer vs.RemoveTable(ctx, qtid2)
defer vs.RemoveTable(ctx, qtid3)
// Add table 1.
assert.NoError(t, vs.AddTable(ctx, qtid1))
// Add table 2.
assert.NoError(t, vs.AddTable(ctx, qtid2))
// Add table 3.
assert.NoError(t, vs.AddTable(ctx, qtid3))
})
t.Run("Shards", func(t *testing.T) {
vs := boltdb.NewVersionStore(db, logger.NopLogger)
qtids := newQualifiedTableIDs(t, qual, 3)
qtid1 := qtids[0]
qtid2 := qtids[1]
qtid3 := qtids[2]
// Add tables.
assert.NoError(t, vs.AddTable(ctx, qtid1))
assert.NoError(t, vs.AddTable(ctx, qtid2))
assert.NoError(t, vs.AddTable(ctx, qtid3))
defer vs.RemoveTable(ctx, qtid1)
defer vs.RemoveTable(ctx, qtid2)
defer vs.RemoveTable(ctx, qtid3)
// Create some shards to insert into the table.
shards := make(dax.VersionedShards, 3)
for i := range shards {
shards[i] = dax.VersionedShard{
Num: dax.ShardNum(i),
Version: i * 2,
}
}
// Add shards to table 1.
{
err := vs.AddShards(ctx, qtid1, shards...)
assert.NoError(t, err)
}
// Add shards to table 2.
{
err := vs.AddShards(ctx, qtid2, shards...)
assert.NoError(t, err)
}
// Fetch a shard and compare.
{
version, found, err := vs.ShardVersion(ctx, qtid1, 2)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, 4, version)
}
// Fetch all shards and compare.
{
shrds, found, err := vs.Shards(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, shards, shrds)
}
// Fetch tables.
{
tblIDs, err := vs.ShardTables(ctx, qual)
assert.NoError(t, err)
exp := dax.TableIDs{qtid1.ID, qtid2.ID, qtid3.ID}
assert.Equal(t, exp, tblIDs)
}
// Remove table 1.
{
shards, partitions, err := vs.RemoveTable(ctx, qtid1)
assert.NoError(t, err)
assert.Equal(t, shards, shards)
assert.Equal(t, dax.VersionedPartitions{}, partitions)
}
// Fetch all shards and compare.
{
shrds, found, err := vs.Shards(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, dax.VersionedShards{}, shrds)
}
})
t.Run("Partitions", func(t *testing.T) {
vs := boltdb.NewVersionStore(db, logger.NopLogger)
// Create some partitions to insert into the table.
partitions := make(dax.VersionedPartitions, 3)
for i := range partitions {
partitions[i] = dax.VersionedPartition{
Num: dax.PartitionNum(i),
Version: i * 2,
}
}
qtids := newQualifiedTableIDs(t, qual, 2)
qtid1 := qtids[0]
qtid2 := qtids[1]
// Add tables.
assert.NoError(t, vs.AddTable(ctx, qtid1))
assert.NoError(t, vs.AddTable(ctx, qtid2))
defer vs.RemoveTable(ctx, qtid1)
defer vs.RemoveTable(ctx, qtid2)
// Add partitions to table 1.
{
err := vs.AddPartitions(ctx, qtid1, partitions...)
assert.NoError(t, err)
}
// Add partitions to table 2.
{
err := vs.AddPartitions(ctx, qtid2, partitions...)
assert.NoError(t, err)
}
// Fetch a partition and compare.
{
version, found, err := vs.PartitionVersion(ctx, qtid1, 2)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, 4, version)
}
// Fetch all partitions and compare.
{
parts, found, err := vs.Partitions(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, partitions, parts)
}
// Fetch tables.
{
tblIDs, err := vs.PartitionTables(ctx, qual)
assert.NoError(t, err)
exp := dax.TableIDs{qtid1.ID, qtid2.ID}
assert.Equal(t, exp, tblIDs)
}
// Remove table 1.
{
shards, partitions, err := vs.RemoveTable(ctx, qtid1)
assert.NoError(t, err)
assert.Equal(t, dax.VersionedShards{}, shards)
assert.Equal(t, partitions, partitions)
}
// Fetch all partitions and compare.
{
parts, found, err := vs.Partitions(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, dax.VersionedPartitions{}, parts)
}
})
t.Run("FieldVersions", func(t *testing.T) {
vs := boltdb.NewVersionStore(db, logger.NopLogger)
qtids := newQualifiedTableIDs(t, qual, 2)
qtid1 := qtids[0]
qtid2 := qtids[1]
// Add tables.
assert.NoError(t, vs.AddTable(ctx, qtid1))
assert.NoError(t, vs.AddTable(ctx, qtid2))
defer vs.RemoveTable(ctx, qtid1)
defer vs.RemoveTable(ctx, qtid2)
// Create some fieldVersions to insert into the table.
fieldVersions := make(dax.VersionedFields, 3)
for i := range fieldVersions {
fieldVersions[i] = dax.VersionedField{
Name: dax.FieldName(fmt.Sprintf("fld-%d", i)),
Version: i * 2,
}
}
// Add fieldVersions to table 1.
{
err := vs.AddFields(ctx, qtid1, fieldVersions...)
assert.NoError(t, err)
}
// Add fieldVersions to table 2.
{
err := vs.AddFields(ctx, qtid2, fieldVersions...)
assert.NoError(t, err)
}
// Fetch a fieldVersion and compare.
{
version, found, err := vs.FieldVersion(ctx, qtid1, dax.FieldName("fld-2"))
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, 4, version)
}
// Fetch all fieldVersions and compare.
{
flds, found, err := vs.Fields(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, fieldVersions, flds)
}
// Fetch tables.
{
tblIDs, err := vs.FieldTables(ctx, qual)
assert.NoError(t, err)
exp := dax.TableIDs{qtid1.ID, qtid2.ID}
assert.Equal(t, exp, tblIDs)
}
// Remove table 1.
{
shards, partitions, err := vs.RemoveTable(ctx, qtid1)
assert.NoError(t, err)
assert.Equal(t, dax.VersionedShards{}, shards)
assert.Equal(t, dax.VersionedPartitions{}, partitions)
}
// Fetch all fieldVersions and compare.
{
flds, found, err := vs.Fields(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, dax.VersionedFields{}, flds)
}
})
t.Run("Copy", func(t *testing.T) {
vs := boltdb.NewVersionStore(db, logger.NopLogger)
qtids := newQualifiedTableIDs(t, qual, 1)
qtid1 := qtids[0]
// Add tables.
assert.NoError(t, vs.AddTable(ctx, qtid1))
defer vs.RemoveTable(ctx, qtid1)
// Create some shards to insert into the table.
shards := make(dax.VersionedShards, 3)
for i := range shards {
shards[i] = dax.VersionedShard{
Num: dax.ShardNum(i),
Version: i * 2,
}
}
// Create some partitions to insert into the table.
partitions := make(dax.VersionedPartitions, 3)
for i := range partitions {
partitions[i] = dax.VersionedPartition{
Num: dax.PartitionNum(i),
Version: i * 2,
}
}
// Create some fieldVersions to insert into the table.
fieldVersions := make(dax.VersionedFields, 3)
for i := range fieldVersions {
fieldVersions[i] = dax.VersionedField{
Name: dax.FieldName(fmt.Sprintf("fld-%d", i)),
Version: i * 2,
}
}
// Add shards to table 1.
{
err := vs.AddShards(ctx, qtid1, shards...)
assert.NoError(t, err)
}
// Add partitions to table 1.
{
err := vs.AddPartitions(ctx, qtid1, partitions...)
assert.NoError(t, err)
}
// Add fieldVersions to table 1.
{
err := vs.AddFields(ctx, qtid1, fieldVersions...)
assert.NoError(t, err)
}
copy, err := vs.Copy(ctx)
assert.NoError(t, err)
// Fetch a shard and compare.
{
version, found, err := copy.ShardVersion(ctx, qtid1, 2)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, 4, version)
}
// Fetch all partitions and compare.
{
parts, found, err := copy.Partitions(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, partitions, parts)
}
// Fetch all fieldVersions and compare.
{
flds, found, err := copy.Fields(ctx, qtid1)
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, fieldVersions, flds)
}
})
}
// newQualifiedTableIDs is a test helper function which generates a slice of n
// qtid. The entries in the slice will be ordered by TableID.
func newQualifiedTableIDs(t *testing.T, qual dax.TableQualifier, n int) []dax.QualifiedTableID {
t.Helper()
qtids := make([]dax.QualifiedTableID, n)
for i := range qtids {
tbl := dax.NewTable("testvstore")
tbl.CreateID()
qtids[i] = dax.NewQualifiedTableID(
qual,
tbl.ID,
)
}
// sort the qtids by ID
sort.Slice(qtids, func(i, j int) bool {
return qtids[i].ID < qtids[j].ID
})
return qtids
}

74
dax/computer/interfaces.go
View file

@ -18,8 +18,12 @@ type Registrar interface {
// These are typically implemented by the WriteLogger client.
type WriteLogService interface {
AppendMessage(bucket string, key string, version int, msg []byte) error
LogReader(bucket string, key string, version int) (io.Reader, io.Closer, error)
LogReader(bucket string, key string, version int) (io.ReadCloser, error)
LogReaderFrom(bucket string, key string, version int, offset int) (io.ReadCloser, error)
DeleteLog(bucket string, key string, version int) error
List(bucket, key string) ([]WriteLogInfo, error)
Lock(bucket, key string) error
Unlock(bucket, key string) error
}
// SnapshotService represents the SnapshotService methods which Computer uses.
@ -28,71 +32,17 @@ type SnapshotService interface {
Read(bucket string, key string, version int) (io.ReadCloser, error)
Write(bucket string, key string, version int, rc io.ReadCloser) error
WriteTo(bucket string, key string, version int, wrTo io.WriterTo) error
List(bucket, key string) ([]SnapInfo, error)
}
// SnapshotReadWriter provides the interface for all snapshot read and writes in
// FeatureBase.
type SnapshotReadWriter interface {
WriteShardData(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int, rc io.ReadCloser) error
ReadShardData(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) (io.ReadCloser, error)
WriteTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int, wrTo io.WriterTo) error
ReadTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) (io.ReadCloser, error)
WriteFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int, wrTo io.WriterTo) error
ReadFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) (io.ReadCloser, error)
// SnapInfo holds metadata about a snapshot.
type SnapInfo struct {
Version int
// Date time.Time
}
// WriteLogWriter provides the interface for all data writes to FeatureBase. After
// data has been written to the local FeatureBase node, the respective interface
// method(s) will be called.
type WriteLogWriter interface {
// CreateTableKeys sends a map of string key to uint64 ID for the table and
// partition provided.
CreateTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int, _ map[string]uint64) error
// DeleteTableKeys deletes all table keys for the table and partition
// provided.
DeleteTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) error
// CreateFieldKeys sends a map of string key to uint64 ID for the table and
// field provided.
CreateFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int, _ map[string]uint64) error
// DeleteTableKeys deletes all field keys for the table and field provided.
DeleteFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) error
// WriteShard sends shard data for the table and shard provided.
WriteShard(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int, msg LogMessage) error
// DeleteShard deletes all data for the table and shard provided.
DeleteShard(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) error
}
// WriteLogReader provides the interface for all reads from the write log.
type WriteLogReader interface {
ShardReader(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) ShardReader
TableKeyReader(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) TableKeyReader
FieldKeyReader(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) FieldKeyReader
}
type TableKeyReader interface {
Open() error
Read() (PartitionKeyMap, error)
Close() error
}
type FieldKeyReader interface {
Open() error
Read() (FieldKeyMap, error)
Close() error
}
type ShardReader interface {
Open() error
Read() (LogMessage, error)
Close() error
}
// WriteLogInfo holds metadata about a write log.
type WriteLogInfo SnapInfo
// LogMessage is implemented by a variety of types which can be serialized as
// messages to the WriteLogger.

162
dax/computer/noop.go
View file

@ -1,162 +0,0 @@
package computer
import (
"context"
"io"
"github.com/featurebasedb/featurebase/v3/dax"
)
// Ensure type implements interface.
var _ WriteLogWriter = (*NopWriteLogWriter)(nil)
// NopWriteLogWriter is a no-op implementation of the WriteLogWriter interface.
type NopWriteLogWriter struct{}
func NewNopWriteLogWriter() *NopWriteLogWriter {
return &NopWriteLogWriter{}
}
func (w *NopWriteLogWriter) CreateTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int, m map[string]uint64) error {
return nil
}
func (w *NopWriteLogWriter) DeleteTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) error {
return nil
}
func (w *NopWriteLogWriter) CreateFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int, m map[string]uint64) error {
return nil
}
func (w *NopWriteLogWriter) DeleteFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) error {
return nil
}
func (w *NopWriteLogWriter) WriteShard(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int, msg LogMessage) error {
return nil
}
func (w *NopWriteLogWriter) DeleteShard(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) error {
return nil
}
// Ensure type implements interface.
var _ WriteLogReader = (*NopWriteLogReader)(nil)
// NopWriteLogReader is a no-op implementation of the WriteLogReader interface.
type NopWriteLogReader struct{}
func NewNopWriteLogReader() *NopWriteLogReader {
return &NopWriteLogReader{}
}
func (w *NopWriteLogReader) TableKeyReader(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) TableKeyReader {
return NewNopTableKeyReader()
}
func (w *NopWriteLogReader) FieldKeyReader(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) FieldKeyReader {
return NewNopFieldKeyReader()
}
func (w *NopWriteLogReader) ShardReader(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) ShardReader {
return NewNopShardReader()
}
////////////////////////////////////////////////
// Ensure type implements interface.
var _ TableKeyReader = &NopTableKeyReader{}
// NopTableKeyReader is a no-op implementation of the TableKeyReader
// interface.
type NopTableKeyReader struct{}
func NewNopTableKeyReader() *NopTableKeyReader {
return &NopTableKeyReader{}
}
func (r *NopTableKeyReader) Open() error { return nil }
func (r *NopTableKeyReader) Read() (PartitionKeyMap, error) {
return PartitionKeyMap{}, io.EOF
}
func (r *NopTableKeyReader) Close() error { return nil }
////////////////////////////////////////////////
// Ensure type implements interface.
var _ FieldKeyReader = &NopFieldKeyReader{}
// NopFieldKeyReader is a no-op implementation of the FieldKeyReader
// interface.
type NopFieldKeyReader struct{}
func NewNopFieldKeyReader() *NopFieldKeyReader {
return &NopFieldKeyReader{}
}
func (r *NopFieldKeyReader) Open() error { return nil }
func (r *NopFieldKeyReader) Read() (FieldKeyMap, error) {
return FieldKeyMap{}, io.EOF
}
func (r *NopFieldKeyReader) Close() error { return nil }
////////////////////////////////////////////////
// Ensure type implements interface.
var _ ShardReader = &NopShardReader{}
// NopShardReader is a no-op implementation of the ShardReader interface.
type NopShardReader struct{}
func NewNopShardReader() *NopShardReader {
return &NopShardReader{}
}
func (r *NopShardReader) Open() error { return nil }
func (r *NopShardReader) Read() (LogMessage, error) {
return nil, io.EOF
}
func (r *NopShardReader) Close() error { return nil }
////////////// SNAPSHOT ////////////////////////
// Ensure type implements interface.
var _ SnapshotReadWriter = &NopSnapshotReadWriter{}
// NopSnapshotReadWriter is a no-op implementation of the SnapshotReadWriter
// interface.
type NopSnapshotReadWriter struct{}
func NewNopSnapshotReadWriter() *NopSnapshotReadWriter {
return &NopSnapshotReadWriter{}
}
func (w *NopSnapshotReadWriter) WriteShardData(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int, rc io.ReadCloser) error {
return nil
}
func (w *NopSnapshotReadWriter) ReadShardData(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) (io.ReadCloser, error) {
return &nopReadCloser{}, nil
}
func (w *NopSnapshotReadWriter) WriteTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int, wrTo io.WriterTo) error {
return nil
}
func (w *NopSnapshotReadWriter) ReadTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) (io.ReadCloser, error) {
return &nopReadCloser{}, nil
}
func (w *NopSnapshotReadWriter) WriteFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int, wrTo io.WriterTo) error {
return nil
}
func (w *NopSnapshotReadWriter) ReadFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) (io.ReadCloser, error) {
return &nopReadCloser{}, nil
}
type nopReadCloser struct{}
func (n *nopReadCloser) Read([]byte) (int, error) { return 0, nil }
func (n *nopReadCloser) Close() error { return nil }

24
dax/computer/service/computer.go
View file

@ -7,16 +7,14 @@ import (
"net/http"
featurebase "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/computer"
mdsclient "github.com/featurebasedb/featurebase/v3/dax/mds/client"
"github.com/featurebasedb/featurebase/v3/dax/snapshotter"
snapshotterclient "github.com/featurebasedb/featurebase/v3/dax/snapshotter/client"
"github.com/featurebasedb/featurebase/v3/dax/writelogger"
writeloggerclient "github.com/featurebasedb/featurebase/v3/dax/writelogger/client"
"github.com/featurebasedb/featurebase/v3/errors"
"github.com/featurebasedb/featurebase/v3/logger"
fbserver "github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/computer"
mdsclient "github.com/featurebasedb/featurebase/v3/dax/mds/client"
"github.com/featurebasedb/featurebase/v3/dax/snapshotter"
"github.com/featurebasedb/featurebase/v3/dax/writelogger"
"github.com/featurebasedb/featurebase/v3/errors"
"github.com/featurebasedb/featurebase/v3/logger"
fbserver "github.com/featurebasedb/featurebase/v3/server"
)
// Ensure type implements interface.
@ -157,7 +155,8 @@ func newCommand(addr dax.Address, cfg CommandConfig) *fbserver.Command {
var writeLoggerImpl computer.WriteLogService
if cfg.ComputerConfig.WriteLogger != "" {
writeLoggerImpl = writeloggerclient.New(dax.Address(cfg.ComputerConfig.WriteLogger))
panic("running separate writelogger is currently unsupported")
// writeLoggerImpl = writeloggerclient.New(dax.Address(cfg.ComputerConfig.WriteLogger))
} else if wlSvc != nil {
writeLoggerImpl = wlSvc
} else {
@ -166,7 +165,8 @@ func newCommand(addr dax.Address, cfg CommandConfig) *fbserver.Command {
var snapshotterImpl computer.SnapshotService
if cfg.ComputerConfig.Snapshotter != "" {
snapshotterImpl = snapshotterclient.New(dax.Address(cfg.ComputerConfig.Snapshotter))
panic("running separate snapshotter is currently unsupported")
// snapshotterImpl = snapshotterclient.New(dax.Address(cfg.ComputerConfig.Snapshotter))
} else if ssSvc != nil {
snapshotterImpl = ssSvc
} else {

94
dax/computer/snapshot.go
View file

@ -1,94 +0,0 @@
package computer
import (
"context"
"io"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/errors"
)
// Ensure type implements interface.
var _ SnapshotReadWriter = &snapshotReadWriter{}
// snapshotReadWriter uses a SnapshotService implementation (which could be, for
// example, an http client or a locally running sub-service) to store its
// snapshots.
type snapshotReadWriter struct {
ss SnapshotService
}
func NewSnapshotReadWriter(ss SnapshotService) *snapshotReadWriter {
return &snapshotReadWriter{
ss: ss,
}
}
func (s *snapshotReadWriter) WriteShardData(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int, rc io.ReadCloser) error {
bucket := partitionBucket(qtid.Key(), partition)
key := shardKey(shard)
if err := s.ss.Write(bucket, key, version, rc); err != nil {
return errors.Wrapf(err, "writing shard data: %s, %d", key, version)
}
return nil
}
func (s *snapshotReadWriter) ReadShardData(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) (io.ReadCloser, error) {
bucket := partitionBucket(qtid.Key(), partition)
key := shardKey(shard)
rc, err := s.ss.Read(bucket, key, version)
if err != nil {
return nil, errors.Wrapf(err, "reading shard data: %s, %s, %d", bucket, key, version)
}
return rc, nil
}
func (s *snapshotReadWriter) WriteTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int, wrTo io.WriterTo) error {
bucket := partitionBucket(qtid.Key(), partition)
key := keysFileName
if err := s.ss.WriteTo(bucket, key, version, wrTo); err != nil {
return errors.Wrapf(err, "writing table keys: %s, %d", key, version)
}
return nil
}
func (s *snapshotReadWriter) ReadTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) (io.ReadCloser, error) {
bucket := partitionBucket(qtid.Key(), partition)
key := keysFileName
rc, err := s.ss.Read(bucket, key, version)
if err != nil {
return nil, errors.Wrapf(err, "reading table keys: %s, %s, %d", bucket, key, version)
}
return rc, nil
}
func (s *snapshotReadWriter) WriteFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int, wrTo io.WriterTo) error {
bucket := fieldBucket(qtid.Key(), field)
key := keysFileName
if err := s.ss.WriteTo(bucket, key, version, wrTo); err != nil {
return errors.Wrapf(err, "writing field keys: %s, %d", key, version)
}
return nil
}
func (s *snapshotReadWriter) ReadFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) (io.ReadCloser, error) {
bucket := fieldBucket(qtid.Key(), field)
key := keysFileName
rc, err := s.ss.Read(bucket, key, version)
if err != nil {
return nil, errors.Wrapf(err, "reading field keys: %s, %s, %d", bucket, key, version)
}
return rc, nil
}

307
dax/computer/writelog.go
View file

@ -1,307 +0,0 @@
package computer
import (
"bufio"
"context"
"encoding/json"
"io"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/errors"
)
// Ensure type implements interface.
var _ WriteLogReader = &writeLogReadWriter{}
var _ WriteLogWriter = &writeLogReadWriter{}
// writeLogReadWriter is an implementation of the WriteLogReader and WriteLogWriter
// interfaces. It uses a WriteLogService implementation (which could be, for
// example, an http client or a locally running sub-service) to store its log
// messages.
type writeLogReadWriter struct {
wls WriteLogService
}
func NewWriteLogReadWriter(wls WriteLogService) *writeLogReadWriter {
return &writeLogReadWriter{
wls: wls,
}
}
func (w *writeLogReadWriter) CreateTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int, m map[string]uint64) error {
msg := PartitionKeyMap{
TableKey: qtid.Key(),
Partition: partition,
StringToID: m,
}
b, err := json.Marshal(msg)
if err != nil {
return errors.Wrap(err, "marshalling partition key map to json")
}
bucket := partitionBucket(qtid.Key(), partition)
if err := w.wls.AppendMessage(bucket, keysFileName, version, b); err != nil {
return errors.Wrapf(err, "appending partition key message: %s, %d", keysFileName, version)
}
return nil
}
func (w *writeLogReadWriter) DeleteTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) error {
bucket := partitionBucket(qtid.Key(), partition)
return w.wls.DeleteLog(bucket, keysFileName, version)
}
func (w *writeLogReadWriter) CreateFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int, m map[string]uint64) error {
msg := FieldKeyMap{
TableKey: qtid.Key(),
Field: field,
StringToID: m,
}
b, err := json.Marshal(msg)
if err != nil {
return errors.Wrap(err, "marshalling field key map to json")
}
bucket := fieldBucket(qtid.Key(), field)
if err := w.wls.AppendMessage(bucket, keysFileName, version, b); err != nil {
return errors.Wrapf(err, "appending field key message: %s, %d", keysFileName, version)
}
return nil
}
func (w *writeLogReadWriter) DeleteFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) error {
bucket := fieldBucket(qtid.Key(), field)
return w.wls.DeleteLog(bucket, keysFileName, version)
}
func (w *writeLogReadWriter) WriteShard(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int, msg LogMessage) error {
b, err := MarshalLogMessage(msg, EncodeTypeJSON)
if err != nil {
return errors.Wrap(err, "marshalling log message")
}
bucket := partitionBucket(qtid.Key(), partition)
shardKey := shardKey(shard)
if err := w.wls.AppendMessage(bucket, shardKey, version, b); err != nil {
return errors.Wrapf(err, "appending shard key message: %s, %d", shardKey, version)
}
return nil
}
func (w *writeLogReadWriter) DeleteShard(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) error {
bucket := partitionBucket(qtid.Key(), partition)
shardKey := shardKey(shard)
return w.wls.DeleteLog(bucket, shardKey, version)
}
////////////////////////////////////////////////
func (w *writeLogReadWriter) TableKeyReader(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) TableKeyReader {
return newTableKeyReader(w.wls, qtid, partition, version)
}
type tableKeyReader struct {
wl WriteLogService
table dax.TableKey
partition dax.PartitionNum
version int
scanner *bufio.Scanner
closer io.Closer
}
func newTableKeyReader(wl WriteLogService, qtid dax.QualifiedTableID, partition dax.PartitionNum, version int) *tableKeyReader {
r := &tableKeyReader{
wl: wl,
table: qtid.Key(),
partition: partition,
version: version,
}
return r
}
func (r *tableKeyReader) Open() error {
bucket := partitionBucket(r.table, r.partition)
reader, closer, err := r.wl.LogReader(bucket, keysFileName, r.version)
if err != nil {
return errors.Wrapf(err, "getting log reader: %s, %s, %d", bucket, keysFileName, r.version)
}
r.closer = closer
r.scanner = bufio.NewScanner(reader)
return nil
}
func (r *tableKeyReader) Read() (PartitionKeyMap, error) {
if r.scanner == nil {
return PartitionKeyMap{}, io.EOF
}
var b []byte
var out PartitionKeyMap
if r.scanner.Scan() {
b = r.scanner.Bytes()
if err := json.Unmarshal(b, &out); err != nil {
return out, err
}
return out, nil
}
if err := r.scanner.Err(); err != nil {
return out, err
}
return out, io.EOF
}
func (r *tableKeyReader) Close() error {
if r.closer != nil {
return r.closer.Close()
}
return nil
}
////////////////////////////////////////////////
func (w *writeLogReadWriter) FieldKeyReader(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName, version int) FieldKeyReader {
return newFieldKeyReader(w.wls, qtid, field, version)
}
type fieldKeyReader struct {
wl WriteLogService
table dax.TableKey
field dax.FieldName
version int
scanner *bufio.Scanner
closer io.Closer
}
func newFieldKeyReader(wl WriteLogService, qtid dax.QualifiedTableID, field dax.FieldName, version int) *fieldKeyReader {
r := &fieldKeyReader{
wl: wl,
table: qtid.Key(),
field: field,
version: version,
}
return r
}
func (r *fieldKeyReader) Open() error {
bucket := fieldBucket(r.table, r.field)
reader, closer, err := r.wl.LogReader(bucket, keysFileName, r.version)
if err != nil {
return errors.Wrapf(err, "getting log reader: %s, %s, %d", bucket, keysFileName, r.version)
}
r.closer = closer
r.scanner = bufio.NewScanner(reader)
return nil
}
func (r *fieldKeyReader) Read() (FieldKeyMap, error) {
if r.scanner == nil {
return FieldKeyMap{}, io.EOF
}
var b []byte
var out FieldKeyMap
if r.scanner.Scan() {
b = r.scanner.Bytes()
if err := json.Unmarshal(b, &out); err != nil {
return out, err
}
return out, nil
}
if err := r.scanner.Err(); err != nil {
return out, err
}
return out, io.EOF
}
func (r *fieldKeyReader) Close() error {
if r.closer != nil {
return r.closer.Close()
}
return nil
}
////////////////////////////////////////////////
func (w *writeLogReadWriter) ShardReader(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) ShardReader {
return newShardReader(w.wls, qtid, partition, shard, version)
}
type shardReader struct {
wl WriteLogService
table dax.TableKey
partition dax.PartitionNum
shard dax.ShardNum
version int
scanner *bufio.Scanner
closer io.Closer
}
func newShardReader(wl WriteLogService, qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, version int) *shardReader {
r := &shardReader{
wl: wl,
table: qtid.Key(),
partition: partition,
shard: shard,
version: version,
}
return r
}
func (r *shardReader) Open() error {
bucket := partitionBucket(r.table, r.partition)
shardKey := shardKey(r.shard)
reader, closer, err := r.wl.LogReader(bucket, shardKey, r.version)
if err != nil {
return errors.Wrapf(err, "getting log reader: %s, %s, %d", bucket, shardKey, r.version)
}
r.closer = closer
r.scanner = bufio.NewScanner(reader)
return nil
}
func (r *shardReader) Read() (LogMessage, error) {
if r.scanner == nil {
return nil, io.EOF
}
if r.scanner.Scan() {
return UnmarshalLogMessage(r.scanner.Bytes())
}
if err := r.scanner.Err(); err != nil {
return nil, err
}
return nil, io.EOF
}
func (r *shardReader) Close() error {
if r.closer != nil {
return r.closer.Close()
}
return nil
}

30
dax/directive.go
View file

@ -1,5 +1,7 @@
package dax
import "context"
// Directive contains the instructions, sent from MDS, which a compute node is
// to follow. A Directive is typically JSON-encoded and POSTed to a compute
// node's `/directive` endpoint.
@ -19,6 +21,10 @@ type Directive struct {
Version uint64 `json:"version"`
}
type DirectiveVersion interface {
Increment(ctx context.Context, delta uint64) (uint64, error)
}
// DirectiveMethod is used to tell the compute node how it should handle the
// Directive.
type DirectiveMethod string
@ -57,9 +63,9 @@ func (d *Directive) Table(qtid QualifiedTableID) (*QualifiedTable, error) {
// compute node is responsible. It assumes that the Directive does not contain
// more than one ComputeRole for the same table; in that case, we would need to
// return the union of Shards.
func (d *Directive) ComputeShards(tbl TableKey) VersionedShards {
func (d *Directive) ComputeShards(tbl TableKey) ShardNums {
if d == nil || d.ComputeRoles == nil {
return VersionedShards{}
return nil
}
for _, cr := range d.ComputeRoles {
@ -68,14 +74,14 @@ func (d *Directive) ComputeShards(tbl TableKey) VersionedShards {
}
}
return VersionedShards{}
return nil
}
// ComputeShardsMap returns a map of table to shards. It assumes that the
// Directive does not contain more than one ComputeRole for the same table; in
// that case, we would need to return the union of Shards.
func (d *Directive) ComputeShardsMap() map[TableKey]VersionedShards {
m := make(map[TableKey]VersionedShards)
func (d *Directive) ComputeShardsMap() map[TableKey]ShardNums {
m := make(map[TableKey]ShardNums)
if d == nil || d.ComputeRoles == nil {
return m
}
@ -91,9 +97,9 @@ func (d *Directive) ComputeShardsMap() map[TableKey]VersionedShards {
// which this translate node is responsible. It assumes that the Directive does
// not contain more than one TranslateRole for the same table; in that case, we
// would need to return the union of Shards.
func (d *Directive) TranslatePartitions(tbl TableKey) VersionedPartitions {
func (d *Directive) TranslatePartitions(tbl TableKey) PartitionNums {
if d == nil || d.TranslateRoles == nil {
return VersionedPartitions{}
return PartitionNums{}
}
for _, tr := range d.TranslateRoles {
@ -101,14 +107,14 @@ func (d *Directive) TranslatePartitions(tbl TableKey) VersionedPartitions {
return tr.Partitions
}
}
return VersionedPartitions{}
return PartitionNums{}
}
// TranslatePartitionsMap returns a map of table to partitions. It assumes that
// the Directive does not contain more than one TranslateRole for the same
// table; in that case, we would need to return the union of Partitions.
func (d *Directive) TranslatePartitionsMap() map[TableKey]VersionedPartitions {
m := make(map[TableKey]VersionedPartitions)
func (d *Directive) TranslatePartitionsMap() map[TableKey]PartitionNums {
m := make(map[TableKey]PartitionNums)
if d == nil || d.TranslateRoles == nil {
return m
}
@ -129,8 +135,8 @@ func (d *Directive) TranslatePartitionsMap() map[TableKey]VersionedPartitions {
// TranslateFieldsMap returns a map of table to fields. It assumes that
// the Directive does not contain more than one TranslateRole for the same
// table; in that case, we would need to return the union of FieldValues.
func (d *Directive) TranslateFieldsMap() map[TableKey]VersionedFields {
m := make(map[TableKey]VersionedFields)
func (d *Directive) TranslateFieldsMap() map[TableKey][]FieldName {
m := make(map[TableKey][]FieldName)
if d == nil || d.TranslateRoles == nil {
return m
}

10
dax/docker-compose.yml
View file

@ -8,9 +8,8 @@ services:
environment:
FEATUREBASE_BIND: 0.0.0.0:8080
FEATUREBASE_VERBOSE: "true"
FEATUREBASE_STORAGE_METHOD: boltdb
FEATUREBASE_STORAGE_DSN: file:/dax-data/mds.boldtb
FEATUREBASE_MDS_RUN: "true"
FEATUREBASE_CONFIG_DATA_DIR: file:/dax-data/mds
ports:
- "8081:8080"
@ -22,7 +21,7 @@ services:
FEATUREBASE_BIND: 0.0.0.0:8080
FEATUREBASE_VERBOSE: "true"
FEATUREBASE_QUERYER_RUN: "true"
FEATUREBASE_QUERYER_CONFIG_MDS_ADDRESS: "mds:8080"
FEATUREBASE_QUERYER_CONFIG_MDS_ADDRESS: "mds:8080/mds"
depends_on:
- mds
ports:
@ -34,8 +33,9 @@ services:
dockerfile: ../Dockerfile-dax-quick
environment:
FEATUREBASE_COMPUTER_RUN: "true"
FEATUREBASE_COMPUTER_CONFIG_MDS_ADDRESS: "mds:8080"
FEATUREBASE_COMPUTER_CONFIG_MDS_ADDRESS: "mds:8080/mds"
FEATUREBASE_COMPUTER_CONFIG_DATA_DIR: /dax-data/computer
FEATUREBASE_COMPUTER_CONFIG_VERBOSE: true
FEATUREBASE_BIND: 0.0.0.0:8080
FEATUREBASE_VERBOSE: "true"
FEATUREBASE_STORAGE_METHOD: boltdb
@ -63,4 +63,4 @@ services:
GEN_USE_SHARD_TRANSACTIONAL_ENDPOINT: "true"
GEN_SOURCE: "custom"
GEN_TARGET: "mds"
GEN_MDS_ADDRESS: "mds:8080"
GEN_MDS_ADDRESS: "mds:8080/mds"

2
dax/inmem/inmem.go
View file

@ -1,2 +0,0 @@
// Package inmem contains the in-memory implementation of the dax interfaces.
package inmem

430
dax/inmem/versionstore.go
View file

@ -1,430 +0,0 @@
package inmem
import (
"context"
"sort"
"sync"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/errors"
)
// Ensure type implements interface.
var _ dax.VersionStore = (*VersionStore)(nil)
// VersionStore manages all version info for shard, table keys, and field keys.
type VersionStore struct {
mu sync.RWMutex
// shards is a map of all shards, by table, by shard number, known to
// contain data.
shards map[dax.TableQualifierKey]map[dax.TableID]map[dax.ShardNum]dax.VersionedShard
// tableKeys is a map of all partitions, by table, by partition number,
// known to contain key data.
tableKeys map[dax.TableQualifierKey]map[dax.TableID]map[dax.PartitionNum]int
// fieldKeys is a map of all fields, by table, known to contain key data.
fieldKeys map[dax.TableQualifierKey]map[dax.TableID]map[dax.FieldName]int
}
// NewVersionStore returns a new instance of VersionStore with default values.
func NewVersionStore() *VersionStore {
return &VersionStore{
shards: make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.ShardNum]dax.VersionedShard),
tableKeys: make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.PartitionNum]int),
fieldKeys: make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.FieldName]int),
}
}
// AddTable adds a table to be managed by VersionStore.
func (s *VersionStore) AddTable(ctx context.Context, qtid dax.QualifiedTableID) error {
s.mu.Lock()
defer s.mu.Unlock()
// This check is clunky; three maps contain the table, but we only check for
// existence in one of them. It also seems weird to check all three, because
// if we get in a state where one of the maps doesn't contain a table that
// the other maps do contain, the state of the data is in question.
if _, found := s.shards[qtid.TableQualifier.Key()][qtid.ID]; found {
return dax.NewErrTableIDExists(qtid)
}
// Initialize the maps in case VersionStore wasn't created with NewVersionStore().
if s.shards == nil {
s.shards = make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.ShardNum]dax.VersionedShard)
}
if s.tableKeys == nil {
s.tableKeys = make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.PartitionNum]int)
}
if s.fieldKeys == nil {
s.fieldKeys = make(map[dax.TableQualifierKey]map[dax.TableID]map[dax.FieldName]int)
}
// shards.
if _, ok := s.shards[qtid.TableQualifier.Key()]; !ok {
s.shards[qtid.TableQualifier.Key()] = make(map[dax.TableID]map[dax.ShardNum]dax.VersionedShard, 0)
}
if _, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]; !ok {
s.shards[qtid.TableQualifier.Key()][qtid.ID] = make(map[dax.ShardNum]dax.VersionedShard, 0)
}
// tableKeys.
if _, ok := s.tableKeys[qtid.TableQualifier.Key()]; !ok {
s.tableKeys[qtid.TableQualifier.Key()] = make(map[dax.TableID]map[dax.PartitionNum]int, 0)
}
if _, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]; !ok {
s.tableKeys[qtid.TableQualifier.Key()][qtid.ID] = make(map[dax.PartitionNum]int, 0)
}
// fieldKeys.
if _, ok := s.fieldKeys[qtid.TableQualifier.Key()]; !ok {
s.fieldKeys[qtid.TableQualifier.Key()] = make(map[dax.TableID]map[dax.FieldName]int, 0)
}
if _, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]; !ok {
s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID] = make(map[dax.FieldName]int, 0)
}
return nil
}
// RemoveTable removes the given table. An error will be returned if the table
// does not exist.
func (s *VersionStore) RemoveTable(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedShards, dax.VersionedPartitions, error) {
s.mu.Lock()
defer s.mu.Unlock()
var foundTable bool
var shards dax.VersionedShards
var partitions dax.VersionedPartitions
var err error
// Remove shards for table.
if s.shards != nil {
if _, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]; ok {
foundTable = true
// Get the shards to return before deleting from map.
shards, _, err = s.shardSlice(qtid)
if err != nil {
return nil, nil, errors.Wrapf(err, "getting shard slice: %s", qtid)
}
// Remove the shards.
delete(s.shards[qtid.TableQualifier.Key()], qtid.ID)
}
}
// Remove tableKeys for table.
if s.tableKeys != nil {
if _, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]; ok {
foundTable = true
// Get the partitions to return before deleting from map.
partitions, _, err = s.partitionSlice(qtid)
if err != nil {
return nil, nil, errors.Wrapf(err, "getting partition slice: %s", qtid)
}
// Remove the tableKeys.
delete(s.tableKeys[qtid.TableQualifier.Key()], qtid.ID)
}
}
// Remove fieldKeys for table.
if s.fieldKeys != nil {
if _, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]; ok {
foundTable = true
// Remove the fieldKeys.
delete(s.fieldKeys[qtid.TableQualifier.Key()], qtid.ID)
}
}
if !foundTable {
return nil, nil, dax.NewErrTableIDDoesNotExist(qtid)
}
return shards, partitions, nil
}
// AddShards adds new shards to be managed by VersionStore. It returns the
// number of shards added or an error.
func (s *VersionStore) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.VersionedShard) error {
s.mu.Lock()
defer s.mu.Unlock()
sh, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]
if !ok {
return dax.NewErrTableIDDoesNotExist(qtid)
}
var n int
for _, shard := range shards {
if _, ok := sh[shard.Num]; !ok {
n++ // TODO: this isn't considering a shard that exists, but the version changes.
}
sh[shard.Num] = shard
}
return nil
}
// Shards returns the list of shards available for the give table. It returns
// false if the table does not exist.
func (s *VersionStore) Shards(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedShards, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.shardSlice(qtid)
}
// shardSlice is an unprotected version of Shards().
func (s *VersionStore) shardSlice(qtid dax.QualifiedTableID) (dax.VersionedShards, bool, error) {
if s.shards == nil {
return nil, false, nil
}
if shardNumMap, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]; ok {
rtn := make(dax.VersionedShards, 0, len(shardNumMap))
for _, shard := range shardNumMap {
rtn = append(rtn, shard)
}
sort.Sort(rtn)
return rtn, true, nil
}
return nil, false, nil
}
// ShardVersion return the current version for the given table/shardNum.
// If a version is not being tracked, it returns a bool value of false.
func (s *VersionStore) ShardVersion(ctx context.Context, qtid dax.QualifiedTableID, shardNum dax.ShardNum) (int, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
t, ok := s.shards[qtid.TableQualifier.Key()][qtid.ID]
if !ok {
return -1, false, nil
}
v, ok := t[shardNum]
if !ok {
return -1, false, nil
}
return v.Version, true, nil
}
func (s *VersionStore) ShardTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) {
s.mu.RLock()
defer s.mu.RUnlock()
qual.Key()
tableIDs := make(dax.TableIDs, 0, len(s.shards[qual.Key()]))
for tableID := range s.shards[qual.Key()] {
tableIDs = append(tableIDs, tableID)
}
return tableIDs, nil
}
// AddPartitions adds new partitions to be managed by VersionStore. It returns
// the number of partitions added or an error.
func (s *VersionStore) AddPartitions(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.VersionedPartition) error {
s.mu.Lock()
defer s.mu.Unlock()
tk, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]
if !ok {
return dax.NewErrTableIDDoesNotExist(qtid)
}
for _, partition := range partitions {
tk[partition.Num] = partition.Version
}
return nil
}
// Partitions returns the list of partitions available for the give table. It
// returns false if the table does not exist.
func (s *VersionStore) Partitions(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedPartitions, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.partitionSlice(qtid)
}
// partitionSlice is an unprotected version of Partitions().
func (s *VersionStore) partitionSlice(qtid dax.QualifiedTableID) (dax.VersionedPartitions, bool, error) {
if s.tableKeys == nil {
return nil, false, nil
}
if partitionNumMap, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]; ok {
rtn := make(dax.VersionedPartitions, 0, len(partitionNumMap))
for partitionNum, version := range partitionNumMap {
rtn = append(rtn, dax.NewVersionedPartition(partitionNum, version))
}
sort.Sort(rtn)
return rtn, true, nil
}
return nil, false, nil
}
// PartitionVersion return the current version for the given table/partitionNum.
// If a version is not being tracked, it returns a bool value of false.
func (s *VersionStore) PartitionVersion(ctx context.Context, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) (int, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
t, ok := s.tableKeys[qtid.TableQualifier.Key()][qtid.ID]
if !ok {
return -1, false, nil
}
v, ok := t[partitionNum]
if !ok {
return -1, false, nil
}
return v, true, nil
}
func (s *VersionStore) PartitionTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) {
s.mu.RLock()
defer s.mu.RUnlock()
tableIDs := make(dax.TableIDs, 0, len(s.tableKeys[qual.Key()]))
for tableName := range s.tableKeys[qual.Key()] {
tableIDs = append(tableIDs, tableName)
}
return tableIDs, nil
}
// AddFields adds new fields to be managed by VersionStore. It returns the
// number of fields added or an error.
func (s *VersionStore) AddFields(ctx context.Context, qtid dax.QualifiedTableID, fields ...dax.VersionedField) error {
s.mu.Lock()
defer s.mu.Unlock()
fk, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]
if !ok {
return dax.NewErrTableIDDoesNotExist(qtid)
}
for _, field := range fields {
fk[field.Name] = field.Version
}
return nil
}
// Fields returns the list of fields available for the give table. It returns
// false if the table does not exist.
func (s *VersionStore) Fields(ctx context.Context, qtid dax.QualifiedTableID) (dax.VersionedFields, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.fieldSlice(qtid)
}
// fieldSlice is an unprotected version of Fields().
func (s *VersionStore) fieldSlice(qtid dax.QualifiedTableID) (dax.VersionedFields, bool, error) {
if s.fieldKeys == nil {
return nil, false, nil
}
if fieldNameMap, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]; ok {
rtn := make(dax.VersionedFields, 0, len(fieldNameMap))
for fieldName, version := range fieldNameMap {
rtn = append(rtn, dax.NewVersionedField(fieldName, version))
}
sort.Sort(rtn)
return rtn, true, nil
}
return nil, false, nil
}
// FieldVersion return the current version for the given table/field.
// If a version is not being tracked, it returns a bool value of false.
func (s *VersionStore) FieldVersion(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName) (int, bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
t, ok := s.fieldKeys[qtid.TableQualifier.Key()][qtid.ID]
if !ok {
return -1, false, nil
}
v, ok := t[field]
if !ok {
return -1, false, nil
}
return v, true, nil
}
func (s *VersionStore) FieldTables(ctx context.Context, qual dax.TableQualifier) (dax.TableIDs, error) {
s.mu.RLock()
defer s.mu.RUnlock()
tableIDs := make(dax.TableIDs, 0, len(s.fieldKeys[qual.Key()]))
for tableID := range s.fieldKeys[qual.Key()] {
tableIDs = append(tableIDs, tableID)
}
return tableIDs, nil
}
// Copy returns a new copy of VersionStore.
func (s *VersionStore) Copy(ctx context.Context) (dax.VersionStore, error) {
s.mu.RLock()
defer s.mu.RUnlock()
new := NewVersionStore()
// shards.
for qkey, tableIDs := range s.shards {
for tableID, shards := range tableIDs {
qual := dax.NewTableQualifier(qkey.OrganizationID(), qkey.DatabaseID())
qtid := dax.NewQualifiedTableID(qual, tableID)
_ = new.AddTable(ctx, qtid)
for shardNum, shard := range shards {
new.shards[qual.Key()][tableID][shardNum] = shard
}
}
}
// tableKeys.
for qkey, tableIDs := range s.tableKeys {
for tableID, partitions := range tableIDs {
qual := dax.NewTableQualifier(qkey.OrganizationID(), qkey.DatabaseID())
qtid := dax.NewQualifiedTableID(qual, tableID)
_ = new.AddTable(ctx, qtid)
for partitionNum, version := range partitions {
new.tableKeys[qual.Key()][tableID][partitionNum] = version
}
}
}
// fieldKeys.
for qkey, tableIDs := range s.fieldKeys {
for tableID, fields := range tableIDs {
qual := dax.NewTableQualifier(qkey.OrganizationID(), qkey.DatabaseID())
qtid := dax.NewQualifiedTableID(qual, tableID)
_ = new.AddTable(ctx, qtid)
for field, version := range fields {
new.fieldKeys[qual.Key()][tableID][field] = version
}
}
}
return new, nil
}

166
dax/inmem/versionstore_test.go
View file

@ -1,166 +0,0 @@
package inmem_test
import (
"context"
"testing"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/inmem"
"github.com/featurebasedb/featurebase/v3/errors"
"github.com/stretchr/testify/assert"
)
func TestVersionStore(t *testing.T) {
orgID := dax.OrganizationID("acme")
dbID := dax.DatabaseID("db1")
tableID := dax.TableID("0000000000000001")
qual := dax.NewTableQualifier(orgID, dbID)
qtid := dax.NewQualifiedTableID(qual, tableID)
invalidQtid := dax.NewQualifiedTableID(qual, dax.TableID("0000000000000000"))
ctx := context.Background()
// Ensure that when using a Schemar not initiated with NewSchemar, the error
// handling works as expected.
t.Run("EmptyVersionStore", func(t *testing.T) {
s := inmem.VersionStore{}
t.Run("GetShardsInvalid", func(t *testing.T) {
sh, ok, err := s.Shards(ctx, invalidQtid)
assert.NoError(t, err)
assert.False(t, ok)
assert.Nil(t, sh)
})
// Add new table.
assert.NoError(t, s.AddTable(ctx, qtid))
})
t.Run("NewVersionStore", func(t *testing.T) {
s := inmem.NewVersionStore()
// Add new table.
assert.NoError(t, s.AddTable(ctx, qtid))
t.Run("AddTableAgain", func(t *testing.T) {
err := s.AddTable(ctx, qtid)
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrTableIDExists))
}
})
t.Run("AddShards", func(t *testing.T) {
err := s.AddShards(ctx, invalidQtid, dax.NewVersionedShard(1, 0))
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist))
}
{
_, ok, err := s.Shards(ctx, invalidQtid)
assert.NoError(t, err)
assert.False(t, ok)
}
// Shards is empty if no shards have been added.
{
sh, ok, err := s.Shards(ctx, qtid)
assert.NoError(t, err)
assert.True(t, ok)
assert.Equal(t, sh, dax.VersionedShards{})
}
// Add the first set of shards (with a duplicate (8)).
{
err := s.AddShards(ctx, qtid,
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(9, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(10, 0),
)
assert.NoError(t, err)
}
{
sh, ok, err := s.Shards(ctx, qtid)
assert.NoError(t, err)
assert.True(t, ok)
assert.Equal(t, dax.VersionedShards{
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(9, 0),
dax.NewVersionedShard(10, 0),
}, sh)
}
// Add another set of shards (with one duplicate (11) and one
// existing (10)).
{
err := s.AddShards(ctx, qtid,
dax.NewVersionedShard(10, 0),
dax.NewVersionedShard(11, 0),
dax.NewVersionedShard(12, 0),
dax.NewVersionedShard(11, 0),
)
assert.NoError(t, err)
}
{
sh, ok, err := s.Shards(ctx, qtid)
assert.NoError(t, err)
assert.True(t, ok)
assert.Equal(t, dax.VersionedShards{
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(9, 0),
dax.NewVersionedShard(10, 0),
dax.NewVersionedShard(11, 0),
dax.NewVersionedShard(12, 0),
}, sh)
}
})
t.Run("RemoveTable", func(t *testing.T) {
shards, partitions, err := s.RemoveTable(ctx, qtid)
assert.NoError(t, err)
assert.Equal(t, dax.VersionedPartitions{}, partitions)
assert.Equal(t, dax.VersionedShards{
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(9, 0),
dax.NewVersionedShard(10, 0),
dax.NewVersionedShard(11, 0),
dax.NewVersionedShard(12, 0),
}, shards)
// Make sure the table was removed.
shards, ok, err := s.Shards(ctx, qtid)
assert.NoError(t, err)
assert.False(t, ok)
assert.Nil(t, shards)
})
})
t.Run("ErrorConditions", func(t *testing.T) {
t.Run("JustSchemar", func(t *testing.T) {
s := inmem.VersionStore{}
shards, partitions, err := s.RemoveTable(ctx, qtid)
assert.Nil(t, shards)
assert.Nil(t, partitions)
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist))
}
})
t.Run("NewSchemar", func(t *testing.T) {
s := inmem.NewVersionStore()
shards, partitions, err := s.RemoveTable(ctx, qtid)
assert.Nil(t, shards)
assert.Nil(t, partitions)
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist))
}
})
})
}

27
dax/mds/client/client.go
View file

@ -10,7 +10,6 @@ import (
"net/http"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/mds/controller"
mdshttp "github.com/featurebasedb/featurebase/v3/dax/mds/http"
"github.com/featurebasedb/featurebase/v3/errors"
"github.com/featurebasedb/featurebase/v3/logger"
@ -49,6 +48,20 @@ func (c *Client) Health() bool {
return true
}
// TODO(tlt): collapse Table into this
func (c *Client) TableByID(ctx context.Context, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) {
return c.Table(ctx, qtid)
}
// TODO(tlt): collapse TableID into this
func (c *Client) TableByName(ctx context.Context, qual dax.TableQualifier, tname dax.TableName) (*dax.QualifiedTable, error) {
qtid, err := c.TableID(ctx, qual, tname)
if err != nil {
return nil, errors.Wrap(err, "getting table id")
}
return c.Table(ctx, qtid)
}
func (c *Client) Table(ctx context.Context, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) {
url := fmt.Sprintf("%s/table", c.address.WithScheme(defaultScheme))
@ -336,11 +349,11 @@ func (c *Client) IngestPartition(ctx context.Context, qtid dax.QualifiedTableID,
return isr.Address, nil
}
func (c *Client) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) ([]controller.ComputeNode, error) {
func (c *Client) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) ([]dax.ComputeNode, error) {
url := fmt.Sprintf("%s/compute-nodes", c.address.WithScheme(defaultScheme))
c.logger.Debugf("ComputeNodes url: %s", url)
var nodes []controller.ComputeNode
var nodes []dax.ComputeNode
req := &mdshttp.ComputeNodesRequest{
Table: qtid,
@ -374,11 +387,11 @@ func (c *Client) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, sh
return cnr.ComputeNodes, nil
}
func (c *Client) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.PartitionNum) ([]controller.TranslateNode, error) {
func (c *Client) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.PartitionNum) ([]dax.TranslateNode, error) {
url := fmt.Sprintf("%s/translate-nodes", c.address.WithScheme(defaultScheme))
c.logger.Debugf("TranslateNodes url: %s", url)
var nodes []controller.TranslateNode
var nodes []dax.TranslateNode
req := &mdshttp.TranslateNodesRequest{
Table: qtid,
@ -414,7 +427,7 @@ func (c *Client) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID,
func (c *Client) RegisterNode(ctx context.Context, node *dax.Node) error {
url := fmt.Sprintf("%s/register-node", c.address.WithScheme(defaultScheme))
c.logger.Debugf("RegisterNode url: %s", url)
c.logger.Debugf("RegisterNode: %s, url: %s", node.Address, url)
req := &mdshttp.RegisterNodeRequest{
Address: node.Address,
@ -437,7 +450,7 @@ func (c *Client) RegisterNode(ctx context.Context, node *dax.Node) error {
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return errors.Errorf("status code: %d: %s", resp.StatusCode, b)
return errors.Errorf("registration request to %s status code: %d: %s", url, resp.StatusCode, b)
}
return nil

125
dax/mds/controller/alpha/director.go
View file

@ -1,125 +0,0 @@
// Package alpha contains inter-service implemenations of interfaces.
package alpha
import (
"context"
"encoding/json"
featurebase "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/mds/controller"
"github.com/featurebasedb/featurebase/v3/errors"
featurebaseserver "github.com/featurebasedb/featurebase/v3/server"
)
// Ensure type implements interface.
var _ controller.Director = (*Director)(nil)
// Director is a direct, service-to-service implementation of the Director
// interface.
type Director struct {
computers map[dax.Address]*featurebaseserver.Command
}
func NewDirector() *Director {
return &Director{
computers: make(map[dax.Address]*featurebaseserver.Command),
}
}
func (d *Director) AddCmd(addr dax.Address, cmd *featurebaseserver.Command) error {
if cmd == nil {
return errors.New(errors.ErrUncoded, "cannot add nil cmd to director")
}
d.computers[addr] = cmd
return nil
}
func (d *Director) api(addr dax.Address) (*featurebase.API, error) {
cmd, found := d.computers[addr]
if !found {
// Address not registered with the Director.
return nil, errors.New(errors.ErrUncoded, "cmd not registered with director")
}
api := cmd.API
if api == nil {
// Command does not have an API.
return nil, errors.New(errors.ErrUncoded, "cmd does not have an api")
}
return api, nil
}
func (d *Director) SendDirective(ctx context.Context, dir *dax.Directive) error {
api, err := d.api(dir.Address)
if err != nil {
return errors.Wrap(err, "getting api from director")
}
ndir, err := marshalUnmarshal(dir)
if err != nil {
return errors.Wrap(err, "marshalUnmarshal")
}
return api.Directive(ctx, ndir)
}
func (d *Director) SendSnapshotShardDataRequest(ctx context.Context, req *dax.SnapshotShardDataRequest) error {
api, err := d.api(req.Address)
if err != nil {
return errors.Wrap(err, "getting api from director")
}
nreq, err := marshalUnmarshal(req)
if err != nil {
return errors.Wrap(err, "marshalUnmarshal")
}
return api.SnapshotShardData(ctx, nreq)
}
func (d *Director) SendSnapshotTableKeysRequest(ctx context.Context, req *dax.SnapshotTableKeysRequest) error {
api, err := d.api(req.Address)
if err != nil {
return errors.Wrap(err, "getting api from director")
}
nreq, err := marshalUnmarshal(req)
if err != nil {
return errors.Wrap(err, "marshalUnmarshal")
}
return api.SnapshotTableKeys(ctx, nreq)
}
func (d *Director) SendSnapshotFieldKeysRequest(ctx context.Context, req *dax.SnapshotFieldKeysRequest) error {
api, err := d.api(req.Address)
if err != nil {
return errors.Wrap(err, "getting api from director")
}
nreq, err := marshalUnmarshal(req)
if err != nil {
return errors.Wrap(err, "marshalUnmarshal")
}
return api.SnapshotFieldKeys(ctx, nreq)
}
// marshalUnmarshal simply marshals anything to json, and then
// unmarshals it. This might seem a bit silly. The reason it exists is
// to exercise the same encode/decode logic that we'd need to if we
// were traversing the network, and guarantee that we aren't sharing
// pointers across API boundaries.
func marshalUnmarshal[K any](a K) (K, error) {
var newA K
abytes, err := json.Marshal(a)
if err != nil {
return newA, errors.Wrap(err, "marshaling directive")
}
if err := json.Unmarshal(abytes, &newA); err != nil {
return newA, errors.Wrap(err, "unmarshaling directive")
}
return newA, nil
}

9
dax/mds/controller/balancer.go
View file

@ -28,6 +28,11 @@ type Balancer interface {
// which are not assigned to any worker, that means the query
// would return incomplete data, so we want to error.
WorkersForJobPrefix(ctx context.Context, prefix string) ([]dax.WorkerInfo, error)
// RemoveJobs is for e.g. when dropping a table remove all jobs
// associated with that table without needing to look up in
// advance which shards or partitions are actually present.
RemoveJobs(ctx context.Context, prefix string) ([]dax.WorkerDiff, error)
}
// Ensure type implements interface.
@ -67,3 +72,7 @@ func (b *NopBalancer) WorkersForJobs(ctx context.Context, jobs []dax.Job) ([]dax
func (b *NopBalancer) WorkersForJobPrefix(ctx context.Context, prefix string) ([]dax.WorkerInfo, error) {
return []dax.WorkerInfo{}, nil
}
func (b *NopBalancer) RemoveJobs(ctx context.Context, prefix string) ([]dax.WorkerDiff, error) {
return nil, nil
}

7
dax/mds/controller/config.go
View file

@ -25,5 +25,12 @@ type Config struct {
// have been registered.
RegistrationBatchTimeout time.Duration
// SnappingTurtleTimeout is the period on which the automatic
// snapshotting routine will run. If performing all the snapshots
// takes longer than this amount of time, snapshotting will run
// continuously. If it finishes before the timeout, it will wait
// until the timeout expires to start another round of snapshots.
SnappingTurtleTimeout time.Duration
Logger logger.Logger
}

521
dax/mds/controller/controller.go
View file

@ -22,9 +22,6 @@ type Controller struct {
// initialization.
mu sync.RWMutex
// versionStore
versionStore dax.VersionStore
// Schemar used by the controller to get table information. The controller
// should NOT call Schemar methods which modify data. Schema mutations are
// made outside of the controller (at this point that happens in MDS).
@ -47,6 +44,8 @@ type Controller struct {
registrationBatchTimeout time.Duration
nodeChan chan *dax.Node
snappingTurtleTimeout time.Duration
snapControl chan struct{}
stopping chan struct{}
logger logger.Logger
@ -69,10 +68,13 @@ func New(cfg Config) *Controller {
poller: dax.NewNopAddressManager(),
logger: logger.NopLogger,
registrationBatchTimeout: cfg.RegistrationBatchTimeout,
nodeChan: make(chan *dax.Node, 10),
snappingTurtleTimeout: cfg.SnappingTurtleTimeout,
snapControl: make(chan struct{}),
nodeChan: make(chan *dax.Node, 10),
stopping: make(chan struct{}),
logger: logger.NopLogger,
}
if cfg.Logger != nil {
@ -81,11 +83,6 @@ func New(cfg Config) *Controller {
switch cfg.StorageMethod {
case "boltdb":
if err := cfg.BoltDB.InitializeBuckets(boltdb.VersionStoreBuckets...); err != nil {
c.logger.Panicf("initializing version store buckets: %v", err)
}
c.versionStore = boltdb.NewVersionStore(cfg.BoltDB, c.logger)
if err := cfg.BoltDB.InitializeBuckets(boltdb.NodeServiceBuckets...); err != nil {
c.logger.Panicf("initializing node service buckets: %v", err)
}
@ -106,14 +103,13 @@ func New(cfg Config) *Controller {
c.Schemar = cfg.Schemar
}
c.registrationBatchTimeout = cfg.RegistrationBatchTimeout
return c
}
// Run starts the node registration goroutine.
// Run starts long running subroutines.
func (c *Controller) Run() error {
go c.nodeRegistrationRoutine(c.nodeChan, c.registrationBatchTimeout)
go c.snappingTurtleRoutine(c.snappingTurtleTimeout, c.snapControl)
return nil
}
@ -518,12 +514,6 @@ func (c *Controller) nodesTranslateReadOrWrite(ctx context.Context, role *dax.Tr
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
// Initialize the partition version to 0.
qtid := j.table().QualifiedTableID()
if err := c.versionStore.AddPartitions(ctx, qtid, dax.NewVersionedPartition(j.partitionNum(), 0)); err != nil {
return nil, false, NewErrInternal(err.Error())
}
}
// Convert the slice of addresses into a slice of addressMethod containing
@ -544,26 +534,14 @@ func (c *Controller) nodesTranslateReadOrWrite(ctx context.Context, role *dax.Tr
for _, worker := range workers {
// covert worker.Jobs []string to map[string][]Partition
translateMap := make(map[dax.TableKey]dax.VersionedPartitions)
translateMap := make(map[dax.TableKey]dax.PartitionNums)
for _, job := range worker.Jobs {
j, err := decodePartition(job)
if err != nil {
return nil, false, NewErrInternal(err.Error())
}
// Get the partition version from the local versionStore.
tkey := j.table()
qtid := tkey.QualifiedTableID()
partitionVersion, found, err := c.versionStore.PartitionVersion(ctx, qtid, j.partitionNum())
if err != nil {
return nil, false, err
} else if !found {
return nil, false, NewErrInternal("partition version not found in cache")
}
translateMap[tkey] = append(translateMap[tkey],
dax.NewVersionedPartition(j.partitionNum(), partitionVersion),
)
translateMap[j.table()] = append(translateMap[j.table()], j.partitionNum())
}
for table, partitions := range translateMap {
@ -705,12 +683,6 @@ func (c *Controller) nodesComputeReadOrWrite(ctx context.Context, role *dax.Comp
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
// Initialize the shard version to 0.
qtid := j.table().QualifiedTableID()
if err := c.versionStore.AddShards(ctx, qtid, dax.NewVersionedShard(j.shardNum(), 0)); err != nil {
return nil, false, NewErrInternal(err.Error())
}
}
// Convert the slice of addresses into a slice of addressMethod containing
@ -737,26 +709,14 @@ func (c *Controller) workersToAssignedNodes(ctx context.Context, workers []dax.W
nodes := []dax.AssignedNode{}
for _, worker := range workers {
// convert worker.Jobs []string to map[TableName][]Shard
computeMap := make(map[dax.TableKey]dax.VersionedShards)
computeMap := make(map[dax.TableKey]dax.ShardNums)
for _, job := range worker.Jobs {
j, err := decodeShard(job)
if err != nil {
return nil, NewErrInternal(err.Error())
}
// Get the shard version from the local versionStore.
tkey := j.table()
qtid := tkey.QualifiedTableID()
shardVersion, found, err := c.versionStore.ShardVersion(ctx, qtid, j.shardNum())
if err != nil {
return nil, err
} else if !found {
return nil, NewErrInternal("shard version not found in cache")
}
computeMap[tkey] = append(computeMap[tkey],
dax.NewVersionedShard(j.shardNum(), shardVersion),
)
computeMap[j.table()] = append(computeMap[j.table()], j.shardNum())
}
for table, shards := range computeMap {
@ -781,31 +741,6 @@ func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable)
c.mu.Lock()
defer c.mu.Unlock()
qtid := qtbl.QualifiedID()
// Add the table to the versionStore.
if err := c.versionStore.AddTable(ctx, qtid); err != nil {
return errors.Wrapf(err, "adding table: %s", qtid)
}
// Add fields which have string keys to the local versionStore.
fieldVersions := make(dax.VersionedFields, 0)
for _, field := range qtbl.Fields {
if !field.StringKeys() {
continue
}
fieldVersions = append(fieldVersions, dax.VersionedField{
Name: field.Name,
Version: 0,
})
}
if len(fieldVersions) > 0 {
if err := c.versionStore.AddFields(ctx, qtid, fieldVersions...); err != nil {
return errors.Wrapf(err, "adding fields: %s, %v", qtid, fieldVersions)
}
}
// If the table is keyed, add partitions to the balancer.
if qtbl.StringKeys() {
// workerSet maintains the set of workers which have a job assignment change
@ -813,15 +748,9 @@ func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable)
workerSet := NewAddressSet()
// Generate the list of partitionsToAdd to be added.
partitionsToAdd := make(dax.VersionedPartitions, qtbl.PartitionN)
partitionsToAdd := make(dax.PartitionNums, qtbl.PartitionN)
for partitionNum := 0; partitionNum < qtbl.PartitionN; partitionNum++ {
partitionsToAdd[partitionNum] = dax.NewVersionedPartition(dax.PartitionNum(partitionNum), 0)
}
// Add partitions to versionStore. Version is intentionally set to 0
// here as this is the initial instance of the partition.
if err := c.versionStore.AddPartitions(ctx, qtid, partitionsToAdd...); err != nil {
return NewErrInternal(err.Error())
partitionsToAdd[partitionNum] = dax.PartitionNum(partitionNum)
}
stringers := make([]fmt.Stringer, 0, len(partitionsToAdd))
@ -856,13 +785,7 @@ func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable)
// and therefore need to be sent an updated Directive.
workerSet := NewAddressSet()
p := dax.NewVersionedPartition(0, 0)
// Add partition 0 to versionStore. Version is intentionally set to 0
// here as this is the initial instance of the partition.
if err := c.versionStore.AddPartitions(ctx, qtid, p); err != nil {
return NewErrInternal(err.Error())
}
p := dax.PartitionNum(0)
// We don't currently use the returned diff, other than to determine
// which worker was affected, because we send the full Directive
@ -898,39 +821,24 @@ func (c *Controller) DropTable(ctx context.Context, qtid dax.QualifiedTableID) e
return errors.Wrapf(err, "table not in schemar: %s", qtid)
}
// Remove the table from the versionStore.
// Since the schemar should be the system of record for the existence of a
// table, if the versionStore is not aware of the table, we just log it and
// continue.
shards, partitions, err := c.versionStore.RemoveTable(ctx, qtid)
if err != nil {
return errors.Wrapf(err, "removing table: %s", qtid)
}
// workerSet maintains the set of workers which have a job assignment change
// and therefore need to be sent an updated Directive.
workerSet := NewAddressSet()
// Remove shards.
for _, s := range shards {
diffs, err := c.ComputeBalancer.RemoveJob(ctx, shard(qtid.Key(), s))
if err != nil {
return errors.Wrap(err, "removing job")
}
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
diffs, err := c.ComputeBalancer.RemoveJobs(ctx, string(qtid.Key()))
if err != nil {
return errors.Wrap(err, "removing jobs")
}
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
// Remove partitions.
for _, p := range partitions {
diffs, err := c.TranslateBalancer.RemoveJob(ctx, partition(qtid.Key(), p))
if err != nil {
return errors.Wrap(err, "removing job")
}
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
diffs, err = c.TranslateBalancer.RemoveJobs(ctx, string(qtid.Key()))
if err != nil {
return errors.Wrap(err, "removing job")
}
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
// Convert the slice of addresses into a slice of addressMethod containing
@ -964,15 +872,10 @@ func (c *Controller) Tables(ctx context.Context, qual dax.TableQualifier, ids ..
// AddShards registers the table/shard combinations with the controller and
// sends the necessary directive.
func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.VersionedShard) error {
func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) error {
c.mu.Lock()
defer c.mu.Unlock()
// Add shards to versionStore.
if err := c.versionStore.AddShards(ctx, qtid, shards...); err != nil {
return errors.Wrapf(err, "adding shards: %s, %v", qtid, shards)
}
// workerSet maintains the set of workers which have a job assignment change
// and therefore need to be sent an updated Directive.
workerSet := NewAddressSet()
@ -988,11 +891,6 @@ func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, s
for _, diff := range diffs {
workerSet.Add(dax.Address(diff.WorkerID))
}
// Initialize the shard version to 0.
if err := c.versionStore.AddShards(ctx, qtid, dax.NewVersionedShard(s.Num, 0)); err != nil {
return NewErrInternal(err.Error())
}
}
// Convert the slice of addresses into a slice of addressMethod containing
@ -1008,7 +906,7 @@ func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, s
// RemoveShards deregisters the table/shard combinations with the controller and
// sends the necessary directives.
func (c *Controller) RemoveShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.VersionedShard) error {
func (c *Controller) RemoveShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) error {
c.mu.Lock()
defer c.mu.Unlock()
@ -1048,7 +946,7 @@ func (c *Controller) sendDirectives(ctx context.Context, addrs ...addressMethod)
return nil
}
directives, err := c.buildDirectives(ctx, addrs, c.versionStore)
directives, err := c.buildDirectives(ctx, addrs)
if err != nil {
return errors.Wrap(err, "building directives")
}
@ -1118,7 +1016,7 @@ func applyAddressMethod(addrs []dax.Address, method dax.DirectiveMethod) []addre
// buildDirectives builds a list of directives for the given addrs (i.e. nodes)
// using information (i.e. current state) from the balancers.
func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod, versionStore dax.VersionStore) ([]*dax.Directive, error) {
func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod) ([]*dax.Directive, error) {
directives := make([]*dax.Directive, len(addrs))
for i, addressMethod := range addrs {
@ -1139,12 +1037,12 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
// computeMap maps a table to a list of shards for that table. We need
// to aggregate them here because the list of jobs from WorkerState()
// can contain a mixture of table/shards.
computeMap := make(map[dax.TableKey][]dax.VersionedShard)
computeMap := make(map[dax.TableKey][]dax.ShardNum)
// translateMap maps a table to a list of partitions for that table. We
// need to aggregate them here because the list of jobs from
// WorkerState() can contain a mixture of table/partitions.
translateMap := make(map[dax.TableKey]dax.VersionedPartitions)
translateMap := make(map[dax.TableKey][]dax.PartitionNum)
// tableSet maintains the set of tables which have a job assignment
// change and therefore need to be included in the Directive schema.
@ -1172,24 +1070,8 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
return nil, errors.Wrapf(err, "decoding shard job: %s", job)
}
// The Shard object decoded from the balancer doesn't
// contain a valid version (because the balancer
// intentionally does not store version information). Here,
// we get the current shard version from the controller's
// cache (i.e. versionStore) and inject that into the Shard
// sent in the directive.
tkey := j.table()
qtid := tkey.QualifiedTableID()
shardVersion, found, err := versionStore.ShardVersion(ctx, qtid, j.shardNum())
if err != nil {
return nil, errors.Wrapf(err, "getting shard version: %s, %d", qtid, j.shardNum())
} else if !found {
return nil, NewErrInternal("shard version not found in cache")
}
computeMap[tkey] = append(computeMap[tkey],
dax.NewVersionedShard(j.shardNum(), shardVersion),
)
computeMap[tkey] = append(computeMap[tkey], j.shardNum())
tableSet.Add(tkey)
}
case dax.RoleTypeTranslate:
@ -1207,24 +1089,8 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
ownsPartition0[j.table()] = struct{}{}
}
// The Partition object decoded from the balancer doesn't
// contain a valid version (because the balancer
// intentionally does not store version information). Here,
// we get the current partition version from the
// controller's cache (i.e. versionStore) and inject that
// into the Partition sent in the directive.
tkey := j.table()
qtid := tkey.QualifiedTableID()
partitionVersion, found, err := versionStore.PartitionVersion(ctx, qtid, j.partitionNum())
if err != nil {
return nil, errors.Wrapf(err, "getting partition version: %s, %d", qtid, j.partitionNum())
} else if !found {
return nil, NewErrInternal("partition version not found in cache")
}
translateMap[tkey] = append(translateMap[tkey],
dax.NewVersionedPartition(j.partitionNum(), partitionVersion),
)
translateMap[tkey] = append(translateMap[tkey], j.partitionNum())
tableSet.Add(tkey)
}
}
@ -1235,7 +1101,7 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
// Because these were encoded as strings in the balancer and may be
// out of order numerically, sort them as integers.
//sort.Slice(v, func(i, j int) bool { return v[i] < v[j] })
sort.Sort(dax.VersionedShards(v))
sort.Sort(dax.ShardNums(v))
d.ComputeRoles = append(d.ComputeRoles, dax.ComputeRole{
TableKey: k,
@ -1247,7 +1113,7 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
for k, v := range translateMap {
// Because these were encoded as strings in the balancer and may be
// out of order numerically, sort them as integers.
sort.Sort(v)
sort.Sort(dax.PartitionNums(v))
d.TranslateRoles = append(d.TranslateRoles, dax.TranslateRole{
TableKey: k,
@ -1269,7 +1135,7 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
return nil, errors.Wrapf(err, "getting table: %s", tkey)
}
fieldVersions := make(dax.VersionedFields, 0)
fieldNames := make([]dax.FieldName, 0)
for _, field := range table.Fields {
if !field.StringKeys() {
continue
@ -1280,26 +1146,16 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod,
continue
}
fieldVersion, found, err := versionStore.FieldVersion(ctx, qtid, field.Name)
if err != nil {
return nil, errors.Wrapf(err, "getting field version: %s, %s", qtid, field)
} else if !found {
return nil, NewErrInternal("field version not found in cache")
}
fieldVersions = append(fieldVersions, dax.VersionedField{
Name: field.Name,
Version: fieldVersion,
})
fieldNames = append(fieldNames, field.Name)
}
if len(fieldVersions) == 0 {
if len(fieldNames) == 0 {
continue
}
d.TranslateRoles = append(d.TranslateRoles, dax.TranslateRole{
TableKey: tkey,
Fields: fieldVersions,
Fields: fieldNames,
})
tableSet.Add(tkey)
@ -1352,48 +1208,10 @@ func (c *Controller) InitializePoller(ctx context.Context) error {
return nil
}
// SnapshotTable snapshots a table.
// SnapshotTable snapshots a table. It might also snapshot everything
// else... no guarantees here, only used in tests as of this writing.
func (c *Controller) SnapshotTable(ctx context.Context, qtid dax.QualifiedTableID) error {
shards, ok, err := c.versionStore.Shards(ctx, qtid)
if err != nil {
return errors.Wrap(err, "getting shards from version store")
} else if !ok {
return errors.New(errors.ErrUncoded, "got false back from versionStore.Shards")
}
for _, shard := range shards {
if err := c.SnapshotShardData(ctx, qtid, shard.Num); err != nil {
return errors.Wrapf(err, "snapshotting shard data: qtid: %s, shard: %d", qtid, shard.Num)
}
}
partitions, ok, err := c.versionStore.Partitions(ctx, qtid)
if err != nil {
return errors.Wrap(err, "getting partitions from version store")
} else if !ok {
return errors.New(errors.ErrUncoded, "got false back from versionStore.Partitions")
}
for _, part := range partitions {
if err := c.SnapshotTableKeys(ctx, qtid, part.Num); err != nil {
return errors.Wrapf(err, "snapshotting table keys: qtid: %s, partition: %d", qtid, part.Num)
}
}
fields, ok, err := c.versionStore.Fields(ctx, qtid)
if err != nil {
return errors.Wrap(err, "getting fields from version store")
} else if !ok {
return errors.New(errors.ErrUncoded, "got false back from versionStore.Fields")
}
for _, fld := range fields {
if fld.Name != "_id" {
if err := c.SnapshotFieldKeys(ctx, qtid, fld.Name); err != nil {
return errors.Wrapf(err, "snapshotting field keys: qtid: %s, field: %s", qtid, fld.Name)
}
}
}
c.snapControl <- struct{}{}
return nil
}
@ -1401,22 +1219,9 @@ func (c *Controller) SnapshotTable(ctx context.Context, qtid dax.QualifiedTableI
// snapshot that shard, then increment its shard version for logs written to the
// WriteLogger.
func (c *Controller) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTableID, shardNum dax.ShardNum) error {
// Confirm table/shard is being tracked; get the current shard.
fromShardVersion, ok, err := c.versionStore.ShardVersion(ctx, qtid, shardNum)
if err != nil {
return errors.Wrapf(err, "getting shard version: %s, %d", qtid, shardNum)
} else if !ok {
return NewErrInternal(
fmt.Sprintf("shard to snapshot not found: %s, %d", qtid, shardNum),
)
}
toShardVersion := fromShardVersion + 1
// Get the node responsible for the shard.
bal := c.ComputeBalancer
job := shard(qtid.Key(), dax.NewVersionedShard(shardNum, -1))
job := shard(qtid.Key(), shardNum)
workers, err := bal.WorkersForJobs(ctx, []dax.Job{dax.Job(job.String())})
if err != nil {
return errors.Wrapf(err, "getting workers for jobs: %s", job)
@ -1428,54 +1233,17 @@ func (c *Controller) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTa
addr := dax.Address(workers[0].ID)
// Make a copy of the controller's versionStore, and update the current
// shard so that the directive sent along with the SnapshotRequest reflects
// the state that we want after a successful snapshot.
versionStoreCopy, err := c.versionStore.Copy(ctx)
if err != nil {
return errors.Wrap(err, "copying version store")
}
if err := versionStoreCopy.AddShards(ctx, qtid,
dax.NewVersionedShard(shardNum, toShardVersion),
); err != nil {
return NewErrInternal(err.Error())
}
// Convert the address into a slice of addressMethod containing the
// appropriate method.
addressMethods := applyAddressMethod([]dax.Address{addr}, dax.DirectiveMethodSnapshot)
var toDirective dax.Directive
if directives, err := c.buildDirectives(ctx, addressMethods, versionStoreCopy); err != nil {
return NewErrInternal(err.Error())
} else if ld := len(directives); ld != 1 {
msg := fmt.Sprintf("buildDirectives returned invalid number of directives: %d", ld)
return NewErrInternal(msg)
} else {
toDirective = *directives[0]
}
// Send the node a snapshot request.
req := &dax.SnapshotShardDataRequest{
Address: addr,
TableKey: qtid.Key(),
ShardNum: shardNum,
FromVersion: fromShardVersion,
ToVersion: toShardVersion,
Directive: toDirective,
Address: addr,
TableKey: qtid.Key(),
ShardNum: shardNum,
}
if err := c.Director.SendSnapshotShardDataRequest(ctx, req); err != nil {
return NewErrInternal(err.Error())
}
// A successful request means the shard version can be incremented.
if err := c.versionStore.AddShards(ctx, qtid,
dax.NewVersionedShard(shardNum, toShardVersion),
); err != nil {
return NewErrInternal(err.Error())
}
return nil
}
@ -1483,22 +1251,9 @@ func (c *Controller) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTa
// partition to snapshot the table keys for that partition, then increment its
// version for logs written to the WriteLogger.
func (c *Controller) SnapshotTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) error {
// Confirm table/shard is being tracked; get the current shard.
fromPartitionVersion, found, err := c.versionStore.PartitionVersion(ctx, qtid, partitionNum)
if err != nil {
return errors.Wrapf(err, "getting partition version: %s, %d", qtid, partitionNum)
} else if !found {
return NewErrInternal(
fmt.Sprintf("partition to snapshot not found: %s, %d", qtid, partitionNum),
)
}
toPartitionVersion := fromPartitionVersion + 1
// Get the node responsible for the partition.
bal := c.TranslateBalancer
job := partition(qtid.Key(), dax.NewVersionedPartition(partitionNum, -1))
job := partition(qtid.Key(), partitionNum)
workers, err := bal.WorkersForJobs(ctx, []dax.Job{dax.Job(job.String())})
if err != nil {
return errors.Wrapf(err, "getting workers for jobs: %s", job)
@ -1510,54 +1265,17 @@ func (c *Controller) SnapshotTableKeys(ctx context.Context, qtid dax.QualifiedTa
addr := dax.Address(workers[0].ID)
// Make a copy of the controller's versionStore, and update the current
// partition so that the directive sent along with the SnapshotRequest
// reflects the state that we want after a successful snapshot.
versionStoreCopy, err := c.versionStore.Copy(ctx)
if err != nil {
return errors.Wrap(err, "copying version store")
}
if err := versionStoreCopy.AddPartitions(ctx, qtid,
dax.NewVersionedPartition(partitionNum, toPartitionVersion),
); err != nil {
return NewErrInternal(err.Error())
}
// Convert the address into a slice of addressMethod containing the
// appropriate method.
addressMethods := applyAddressMethod([]dax.Address{addr}, dax.DirectiveMethodSnapshot)
var toDirective dax.Directive
if directives, err := c.buildDirectives(ctx, addressMethods, versionStoreCopy); err != nil {
return NewErrInternal(err.Error())
} else if ld := len(directives); ld != 1 {
msg := fmt.Sprintf("buildDirectives returned invalid number of directives: %d", ld)
return NewErrInternal(msg)
} else {
toDirective = *directives[0]
}
// Send the node a snapshot request.
req := &dax.SnapshotTableKeysRequest{
Address: addr,
TableKey: qtid.Key(),
PartitionNum: partitionNum,
FromVersion: fromPartitionVersion,
ToVersion: toPartitionVersion,
Directive: toDirective,
}
if err := c.Director.SendSnapshotTableKeysRequest(ctx, req); err != nil {
return NewErrInternal(err.Error())
}
// A successful request means the partition version can be incremented.
if err := c.versionStore.AddPartitions(ctx, qtid,
dax.NewVersionedPartition(partitionNum, toPartitionVersion),
); err != nil {
return NewErrInternal(err.Error())
}
return nil
}
@ -1565,23 +1283,11 @@ func (c *Controller) SnapshotTableKeys(ctx context.Context, qtid dax.QualifiedTa
// to snapshot the keys for that field, then increment its version for logs
// written to the WriteLogger.
func (c *Controller) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName) error {
// Confirm table/field is being tracked; get the current field.
fromFieldVersion, ok, err := c.versionStore.FieldVersion(ctx, qtid, field)
if err != nil {
return errors.Wrapf(err, "getting field version: %s, %s", qtid, field)
} else if !ok {
return NewErrInternal(
fmt.Sprintf("field to snapshot not found: %s, %s", qtid, field),
)
}
toFieldVersion := fromFieldVersion + 1
// Get the node responsible for the field.
bal := c.TranslateBalancer
// Field translation is currently handled by partition 0.
partitionNum := dax.PartitionNum(0)
job := partition(qtid.Key(), dax.NewVersionedPartition(partitionNum, -1))
job := partition(qtid.Key(), partitionNum)
workers, err := bal.WorkersForJobs(ctx, []dax.Job{dax.Job(job.String())})
if err != nil {
@ -1594,63 +1300,26 @@ func (c *Controller) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTa
addr := dax.Address(workers[0].ID)
// Make a copy of the controller's versionStore, and update the current
// field so that the directive sent along with the SnapshotRequest reflects
// the state that we want after a successful snapshot.
versionStoreCopy, err := c.versionStore.Copy(ctx)
if err != nil {
return errors.Wrap(err, "copying version store")
}
if err := versionStoreCopy.AddFields(ctx, qtid,
dax.NewVersionedField(field, toFieldVersion),
); err != nil {
return NewErrInternal(err.Error())
}
// Convert the address into a slice of addressMethod containing the
// appropriate method.
addressMethods := applyAddressMethod([]dax.Address{addr}, dax.DirectiveMethodSnapshot)
var toDirective dax.Directive
if directives, err := c.buildDirectives(ctx, addressMethods, versionStoreCopy); err != nil {
return NewErrInternal(err.Error())
} else if ld := len(directives); ld != 1 {
msg := fmt.Sprintf("buildDirectives returned invalid number of directives: %d", ld)
return NewErrInternal(msg)
} else {
toDirective = *directives[0]
}
// Send the node a snapshot request.
req := &dax.SnapshotFieldKeysRequest{
Address: addr,
TableKey: qtid.Key(),
Field: field,
FromVersion: fromFieldVersion,
ToVersion: toFieldVersion,
Directive: toDirective,
Address: addr,
TableKey: qtid.Key(),
Field: field,
}
if err := c.Director.SendSnapshotFieldKeysRequest(ctx, req); err != nil {
return NewErrInternal(err.Error())
}
// A successful request means the field version can be incremented.
if err := c.versionStore.AddFields(ctx, qtid,
dax.NewVersionedField(field, toFieldVersion),
); err != nil {
return NewErrInternal(err.Error())
}
return nil
}
/////////////
func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards dax.ShardNums, isWrite bool) ([]ComputeNode, error) {
func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards dax.ShardNums, isWrite bool) ([]dax.ComputeNode, error) {
inRole := &dax.ComputeRole{
TableKey: qtid.Key(),
Shards: dax.NewVersionedShards(shards...),
Shards: shards,
}
nodes, err := c.Nodes(ctx, inRole, isWrite)
@ -1658,7 +1327,7 @@ func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID
return nil, errors.Wrap(err, "getting compute nodes")
}
computeNodes := make([]ComputeNode, 0)
computeNodes := make([]dax.ComputeNode, 0)
for _, node := range nodes {
role, ok := node.Role.(*dax.ComputeRole)
@ -1668,20 +1337,20 @@ func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID
return nil, NewErrInternal("not a compute node")
}
computeNodes = append(computeNodes, ComputeNode{
computeNodes = append(computeNodes, dax.ComputeNode{
Address: node.Address,
Table: role.TableKey,
Shards: role.Shards.Nums(),
Shards: role.Shards,
})
}
return computeNodes, nil
}
func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions dax.PartitionNums, isWrite bool) ([]TranslateNode, error) {
func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions dax.PartitionNums, isWrite bool) ([]dax.TranslateNode, error) {
inRole := &dax.TranslateRole{
TableKey: qtid.Key(),
Partitions: dax.NewVersionedPartitions(partitions...),
Partitions: partitions,
}
nodes, err := c.Nodes(ctx, inRole, isWrite)
@ -1689,7 +1358,7 @@ func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTable
return nil, errors.Wrap(err, "getting translate nodes")
}
translateNodes := make([]TranslateNode, 0)
translateNodes := make([]dax.TranslateNode, 0)
for _, node := range nodes {
role, ok := node.Role.(*dax.TranslateRole)
@ -1699,10 +1368,10 @@ func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTable
return nil, NewErrInternal("not a translate node")
}
translateNodes = append(translateNodes, TranslateNode{
translateNodes = append(translateNodes, dax.TranslateNode{
Address: node.Address,
Table: role.TableKey,
Partitions: role.Partitions.Nums(),
Partitions: role.Partitions,
})
}
@ -1759,35 +1428,15 @@ func (c *Controller) CreateField(ctx context.Context, qtid dax.QualifiedTableID,
// and therefore need to be sent an updated Directive.
workerSet := NewAddressSet()
// If the field has string keys, add it to the local versionStore.
if fld.StringKeys() {
fieldVersion := dax.VersionedField{
Name: fld.Name,
Version: 0,
}
if err := c.versionStore.AddFields(ctx, qtid, fieldVersion); err != nil {
return errors.Wrapf(err, "adding fields: %s, %s", qtid, fieldVersion)
}
// Get the worker(s) responsible for partition 0.
job := partition(qtid.Key(), 0).String()
workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)})
if err != nil {
return errors.Wrapf(err, "getting workers for job: %s", job)
}
// Get the worker responsible for partition 0, which handles field key
// translation. Be sure to get the current version.
if v, found, err := c.versionStore.PartitionVersion(ctx, qtid, 0); err != nil {
return errors.Wrapf(err, "getting partition version: %s/0", qtid)
} else if found {
// Get the worker(s) responsible for partition 0.
job := partition(qtid.Key(), dax.VersionedPartition{
Num: 0,
Version: v,
}).String()
workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)})
if err != nil {
return errors.Wrapf(err, "getting workers for job: %s", job)
}
for _, w := range workers {
workerSet.Add(dax.Address(w.ID))
}
for _, w := range workers {
workerSet.Add(dax.Address(w.ID))
}
// Get the list of workers responsible for shard data for this table.
@ -1826,27 +1475,15 @@ func (c *Controller) DropField(ctx context.Context, qtid dax.QualifiedTableID, f
// and therefore need to be sent an updated Directive.
workerSet := NewAddressSet()
// If the field has string keys, remove it from the local versionStore.
// TODO: implement RemoveField() on VersionStore interface.
// Get the worker(s) responsible for partition 0.
job := partition(qtid.Key(), 0).String()
workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)})
if err != nil {
return errors.Wrapf(err, "getting workers for job: %s", job)
}
// Get the worker responsible for partition 0, which handles field key
// translation. Be sure to get the current version.
if v, found, err := c.versionStore.PartitionVersion(ctx, qtid, 0); err != nil {
return errors.Wrapf(err, "getting partition version: %s/0", qtid)
} else if found {
// Get the worker(s) responsible for partition 0.
job := partition(qtid.Key(), dax.VersionedPartition{
Num: 0,
Version: v,
}).String()
workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)})
if err != nil {
return errors.Wrapf(err, "getting workers for job: %s", job)
}
for _, w := range workers {
workerSet.Add(dax.Address(w.ID))
}
for _, w := range workers {
workerSet.Add(dax.Address(w.ID))
}
// Get the list of workers responsible for shard data for this table.

446
dax/mds/controller/controller_test.go
View file

@ -108,19 +108,8 @@ func TestController(t *testing.T) {
exp = []*dax.Directive{}
assert.Equal(t, exp, director.flush())
// Add the same non-keyed table again.
err := con.CreateTable(ctx, tbl0)
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrTableIDExists))
}
exp = []*dax.Directive{}
assert.Equal(t, exp, director.flush())
// Add a shard.
assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(),
dax.NewVersionedShard(0, 0),
))
assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), 0))
exp = []*dax.Directive{
{
@ -132,9 +121,7 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
},
Shards: dax.NewShardNums(0),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -185,13 +172,7 @@ func TestController(t *testing.T) {
assert.Equal(t, exp, director.flush())
// Add more shards.
assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
))
assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), dax.NewShardNums(1, 2, 3, 5, 8)...))
exp = []*dax.Directive{
{
@ -203,10 +184,7 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(3, 0),
},
Shards: dax.NewShardNums(0, 3),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -221,10 +199,7 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(5, 0),
},
Shards: dax.NewShardNums(1, 5),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -239,10 +214,7 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(2, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -257,12 +229,7 @@ func TestController(t *testing.T) {
assert.NoError(t, con.CreateTable(ctx, tbl1))
// Add more shards.
assert.NoError(t, con.AddShards(ctx, tbl1.QualifiedID(),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(13, 0),
))
assert.NoError(t, con.AddShards(ctx, tbl1.QualifiedID(), dax.NewShardNums(3, 5, 8, 13)...))
exp = []*dax.Directive{
{
@ -275,17 +242,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 13),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(3, 0),
},
Shards: dax.NewShardNums(0, 3),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -301,16 +262,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(5, 0),
},
Shards: dax.NewShardNums(5),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(5, 0),
},
Shards: dax.NewShardNums(1, 5),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -326,16 +282,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(8),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(2, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -358,18 +309,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 13),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(3, 0),
},
Shards: dax.NewShardNums(0, 1, 3),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -385,18 +329,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(5, 8),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(2, 5, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -419,23 +356,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 5, 8, 13),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(0, 1, 2, 3, 5, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -470,23 +395,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 5, 8, 13),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(0, 1, 2, 3, 5, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -496,10 +409,7 @@ func TestController(t *testing.T) {
assert.Equal(t, exp, director.flush())
// Remove shards.
assert.NoError(t, con.RemoveShards(ctx, tbl0.QualifiedID(),
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(5, 0),
))
assert.NoError(t, con.RemoveShards(ctx, tbl0.QualifiedID(), dax.NewShardNums(2, 5)...))
exp = []*dax.Directive{
{
@ -512,21 +422,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 5, 8, 13),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(0, 1, 3, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -538,10 +438,7 @@ func TestController(t *testing.T) {
// Remove shards, one which does not exist.
// Currently that doesn't result in an error, it simply no-ops on trying
// to remove 99.
assert.NoError(t, con.RemoveShards(ctx, tbl0.QualifiedID(),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(99, 0),
))
assert.NoError(t, con.RemoveShards(ctx, tbl0.QualifiedID(), dax.NewShardNums(3, 99)...))
exp = []*dax.Directive{
{
@ -554,20 +451,11 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 5, 8, 13),
},
{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(8, 0),
},
Shards: dax.NewShardNums(0, 1, 8),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -589,12 +477,7 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{
{
TableKey: tbl1.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(5, 0),
dax.NewVersionedShard(8, 0),
dax.NewVersionedShard(13, 0),
},
Shards: dax.NewShardNums(3, 5, 8, 13),
},
},
TranslateRoles: []dax.TranslateRole{},
@ -604,7 +487,7 @@ func TestController(t *testing.T) {
assert.Equal(t, exp, director.flush())
// Remove a node which doesn't exist.
err = con.DeregisterNodes(ctx, "invalidNode")
err := con.DeregisterNodes(ctx, "invalidNode")
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrNodeDoesNotExist))
}
@ -683,17 +566,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(2, 0),
dax.NewVersionedPartition(3, 0),
dax.NewVersionedPartition(4, 0),
dax.NewVersionedPartition(5, 0),
dax.NewVersionedPartition(6, 0),
dax.NewVersionedPartition(7, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0, 1, 2, 3, 4, 5, 6, 7),
},
},
Version: 2,
@ -720,13 +594,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(2, 0),
dax.NewVersionedPartition(3, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0, 1, 2, 3),
},
},
Version: 3,
@ -740,13 +609,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(4, 0),
dax.NewVersionedPartition(5, 0),
dax.NewVersionedPartition(6, 0),
dax.NewVersionedPartition(7, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(4, 5, 6, 7),
},
},
Version: 4,
@ -772,12 +636,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(2, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0, 1, 2),
},
},
Version: 5,
@ -791,12 +651,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(4, 0),
dax.NewVersionedPartition(5, 0),
dax.NewVersionedPartition(6, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(4, 5, 6),
},
},
Version: 6,
@ -810,11 +666,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(3, 0),
dax.NewVersionedPartition(7, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(3, 7),
},
},
Version: 7,
@ -842,25 +695,12 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl1.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(4, 0),
dax.NewVersionedPartition(7, 0),
dax.NewVersionedPartition(10, 0),
dax.NewVersionedPartition(13, 0),
dax.NewVersionedPartition(16, 0),
dax.NewVersionedPartition(19, 0),
dax.NewVersionedPartition(22, 0),
},
TableKey: tbl1.Key(),
Partitions: dax.NewPartitionNums(1, 4, 7, 10, 13, 16, 19, 22),
},
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(2, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0, 1, 2),
},
},
Version: 8,
@ -875,25 +715,12 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl1.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(2, 0),
dax.NewVersionedPartition(5, 0),
dax.NewVersionedPartition(8, 0),
dax.NewVersionedPartition(11, 0),
dax.NewVersionedPartition(14, 0),
dax.NewVersionedPartition(17, 0),
dax.NewVersionedPartition(20, 0),
dax.NewVersionedPartition(23, 0),
},
TableKey: tbl1.Key(),
Partitions: dax.NewPartitionNums(2, 5, 8, 11, 14, 17, 20, 23),
},
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(4, 0),
dax.NewVersionedPartition(5, 0),
dax.NewVersionedPartition(6, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(4, 5, 6),
},
},
Version: 9,
@ -908,24 +735,12 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl1.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(3, 0),
dax.NewVersionedPartition(6, 0),
dax.NewVersionedPartition(9, 0),
dax.NewVersionedPartition(12, 0),
dax.NewVersionedPartition(15, 0),
dax.NewVersionedPartition(18, 0),
dax.NewVersionedPartition(21, 0),
},
TableKey: tbl1.Key(),
Partitions: dax.NewPartitionNums(0, 3, 6, 9, 12, 15, 18, 21),
},
{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(3, 0),
dax.NewVersionedPartition(7, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(3, 7),
},
},
Version: 10,
@ -947,17 +762,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl1.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(4, 0),
dax.NewVersionedPartition(7, 0),
dax.NewVersionedPartition(10, 0),
dax.NewVersionedPartition(13, 0),
dax.NewVersionedPartition(16, 0),
dax.NewVersionedPartition(19, 0),
dax.NewVersionedPartition(22, 0),
},
TableKey: tbl1.Key(),
Partitions: dax.NewPartitionNums(1, 4, 7, 10, 13, 16, 19, 22),
},
},
Version: 11,
@ -971,17 +777,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl1.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(2, 0),
dax.NewVersionedPartition(5, 0),
dax.NewVersionedPartition(8, 0),
dax.NewVersionedPartition(11, 0),
dax.NewVersionedPartition(14, 0),
dax.NewVersionedPartition(17, 0),
dax.NewVersionedPartition(20, 0),
dax.NewVersionedPartition(23, 0),
},
TableKey: tbl1.Key(),
Partitions: dax.NewPartitionNums(2, 5, 8, 11, 14, 17, 20, 23),
},
},
Version: 12,
@ -995,17 +792,8 @@ func TestController(t *testing.T) {
ComputeRoles: []dax.ComputeRole{},
TranslateRoles: []dax.TranslateRole{
{
TableKey: tbl1.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(3, 0),
dax.NewVersionedPartition(6, 0),
dax.NewVersionedPartition(9, 0),
dax.NewVersionedPartition(12, 0),
dax.NewVersionedPartition(15, 0),
dax.NewVersionedPartition(18, 0),
dax.NewVersionedPartition(21, 0),
},
TableKey: tbl1.Key(),
Partitions: dax.NewPartitionNums(0, 3, 6, 9, 12, 15, 18, 21),
},
},
Version: 13,
@ -1019,15 +807,6 @@ func TestController(t *testing.T) {
assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist))
}
// Add shards to a table which doesn't exist.
err = con.AddShards(ctx, invalidQtid,
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(2, 0),
)
if assert.Error(t, err) {
assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist))
}
// Register an invalid node.
nodeX := &dax.Node{
Address: "",
@ -1082,14 +861,7 @@ func TestController(t *testing.T) {
assert.NoError(t, con.CreateTable(ctx, tbl0))
// Add shards.
assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(),
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(3, 0),
dax.NewVersionedShard(11, 0),
dax.NewVersionedShard(12, 0),
))
assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), 0, 1, 2, 3, 11, 12))
t.Run("ComputeRole", func(t *testing.T) {
tests := []struct {
@ -1100,27 +872,21 @@ func TestController(t *testing.T) {
{
role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.NewVersionedShards(0, 1, 2, 3),
Shards: dax.NewShardNums(0, 1, 2, 3),
},
exp: []dax.AssignedNode{
{
Address: node0.Address,
Role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(0, 0),
dax.NewVersionedShard(2, 0),
},
Shards: dax.NewShardNums(0, 2),
},
},
{
Address: node1.Address,
Role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(3, 0),
},
Shards: dax.NewShardNums(1, 3),
},
},
},
@ -1128,16 +894,14 @@ func TestController(t *testing.T) {
{
role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.NewVersionedShards(1),
Shards: dax.NewShardNums(1),
},
exp: []dax.AssignedNode{
{
Address: node1.Address,
Role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(1, 0),
},
Shards: dax.NewShardNums(1),
},
},
},
@ -1146,7 +910,7 @@ func TestController(t *testing.T) {
// Add unassigned shards.
role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.NewVersionedShards(1, 888, 889),
Shards: dax.NewShardNums(1, 888, 889),
},
isWrite: true,
exp: []dax.AssignedNode{
@ -1154,19 +918,14 @@ func TestController(t *testing.T) {
Address: node0.Address,
Role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(888, 0),
},
Shards: dax.NewShardNums(888),
},
},
{
Address: node1.Address,
Role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(1, 0),
dax.NewVersionedShard(889, 0),
},
Shards: dax.NewShardNums(1, 889),
},
},
},
@ -1175,17 +934,14 @@ func TestController(t *testing.T) {
// Ensure shards are not returned sorted as strings.
role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.NewVersionedShards(2, 11),
Shards: dax.NewShardNums(2, 11),
},
exp: []dax.AssignedNode{
{
Address: node0.Address,
Role: &dax.ComputeRole{
TableKey: tbl0.Key(),
Shards: dax.VersionedShards{
dax.NewVersionedShard(2, 0),
dax.NewVersionedShard(11, 0),
},
Shards: dax.NewShardNums(2, 11),
},
},
},
@ -1209,75 +965,55 @@ func TestController(t *testing.T) {
}{
{
role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, -1),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0),
},
isWrite: true,
exp: []dax.AssignedNode{
{
Address: node0.Address,
Role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0),
},
},
},
},
{
role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, -1),
dax.NewVersionedPartition(1, -1),
dax.NewVersionedPartition(2, -1),
dax.NewVersionedPartition(3, -1),
dax.NewVersionedPartition(999, -1),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0, 1, 2, 3, 999),
},
isWrite: false,
exp: []dax.AssignedNode{
{
Address: node0.Address,
Role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(0, 0),
dax.NewVersionedPartition(2, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(0, 2),
},
},
{
Address: node1.Address,
Role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(1, 0),
dax.NewVersionedPartition(3, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(1, 3),
},
},
},
},
{
role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(1, -1),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(1),
},
isWrite: false,
exp: []dax.AssignedNode{
{
Address: node1.Address,
Role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(1, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(1),
},
},
},
@ -1285,22 +1021,16 @@ func TestController(t *testing.T) {
{
// Ensure partitions are not returned sorted as strings.
role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(2, -1),
dax.NewVersionedPartition(10, -1),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(2, 10),
},
isWrite: false,
exp: []dax.AssignedNode{
{
Address: node0.Address,
Role: &dax.TranslateRole{
TableKey: tbl0.Key(),
Partitions: dax.VersionedPartitions{
dax.NewVersionedPartition(2, 0),
dax.NewVersionedPartition(10, 0),
},
TableKey: tbl0.Key(),
Partitions: dax.NewPartitionNums(2, 10),
},
},
},

1
dax/mds/controller/http/director.go
View file

@ -100,7 +100,6 @@ func (d *Director) SendDirective(ctx context.Context, dir *dax.Directive) error
func (d *Director) SendSnapshotShardDataRequest(ctx context.Context, req *dax.SnapshotShardDataRequest) error {
url := fmt.Sprintf("%s/%s/shard-data", req.Address.WithScheme("http"), d.snapshotRequestPath)
d.logger.Printf("SEND HTTP snapshot shard data request to: %s\n", url)
// Encode the request.
postBody, err := json.Marshal(req)

67
dax/mds/controller/naive/balancer.go
View file

@ -55,15 +55,18 @@ type WorkerJobService interface {
CreateJobs(ctx context.Context, balancerName string, worker dax.Worker, job ...dax.Job) error
DeleteJob(ctx context.Context, balancerName string, worker dax.Worker, job dax.Job) error
DeleteJobs(ctx context.Context, balancerName, prefix string) (InternalDiffs, error)
JobCounts(ctx context.Context, balancerName string, worker ...dax.Worker) (map[dax.Worker]int, error)
ListJobs(ctx context.Context, balancerName string, worker dax.Worker) (dax.Jobs, error)
}
// TODO: I don't think all these method names need "Free" in them.
type FreeJobService interface {
CreateFreeJobs(ctx context.Context, balancerName string, job ...dax.Job) error
DeleteFreeJob(ctx context.Context, balancerName string, job dax.Job) error
ListFreeJobs(ctx context.Context, balancerName string) (dax.Jobs, error)
MergeFreeJobs(ctx context.Context, balancerName string, jobs dax.Jobs) error
DeleteFreeJobs(ctx context.Context, balancerName, prefix string) error
}
// New returns a new instance of Balancer.
@ -90,15 +93,15 @@ func (b *Balancer) AddWorker(ctx context.Context, worker fmt.Stringer) ([]dax.Wo
return nil, errors.Wrap(err, "adding worker")
}
return diff.output(), nil
return diff.Output(), nil
}
func (b *Balancer) addWorker(ctx context.Context, worker dax.Worker) (internalDiffs, error) {
func (b *Balancer) addWorker(ctx context.Context, worker dax.Worker) (InternalDiffs, error) {
// If this worker already exists, don't do anything.
if exists, err := b.current.WorkerExists(ctx, b.name, worker); err != nil {
return nil, errors.Wrap(err, "checking if worker exists")
} else if exists {
return internalDiffs{}, nil
return InternalDiffs{}, nil
}
if err := b.current.CreateWorker(ctx, b.name, worker); err != nil {
@ -133,15 +136,15 @@ func (b *Balancer) RemoveWorker(ctx context.Context, worker fmt.Stringer) ([]dax
return nil, errors.Wrap(err, "removing worker")
}
return diff.output(), nil
return diff.Output(), nil
}
func (b *Balancer) removeWorker(ctx context.Context, worker dax.Worker) (internalDiffs, error) {
func (b *Balancer) removeWorker(ctx context.Context, worker dax.Worker) (InternalDiffs, error) {
// If this worker doesn't exist, don't do anything else.
if exists, err := b.current.WorkerExists(ctx, b.name, worker); err != nil {
return nil, errors.Wrap(err, "checking if worker exists")
} else if !exists {
return internalDiffs{}, nil
return InternalDiffs{}, nil
}
jobs, err := b.current.ListJobs(ctx, b.name, worker)
@ -162,9 +165,9 @@ func (b *Balancer) removeWorker(ctx context.Context, worker dax.Worker) (interna
// Even though this may not be useful to the caller (for example, in the
// case where the worker has died and no longer exists), return the diffs
// which represent the removal of jobs from the worker.
diff := newInternalDiffs()
diff := NewInternalDiffs()
for _, job := range jobs {
diff.removed(worker, job)
diff.Removed(worker, job)
}
return diff, nil
@ -198,10 +201,10 @@ func (b *Balancer) AddJobs(ctx context.Context, jobs ...fmt.Stringer) ([]dax.Wor
return nil, errors.Wrap(err, "adding job")
}
return diff.output(), nil
return diff.Output(), nil
}
func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (internalDiffs, error) {
func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (InternalDiffs, error) {
if cnt, err := b.current.WorkerCount(ctx, b.name); err != nil {
return nil, errors.Wrap(err, "getting worker count")
} else if cnt == 0 {
@ -210,7 +213,7 @@ func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (internalDiffs,
}
// TODO: we might want to inform the user that a job is in the free list
// because there are no workers.
return internalDiffs{}, nil
return InternalDiffs{}, nil
}
workerJobs, err := b.current.WorkersJobs(ctx, b.name)
@ -229,7 +232,7 @@ func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (internalDiffs,
jobCounts[v.ID] = len(v.Jobs)
}
diffs := newInternalDiffs()
diffs := NewInternalDiffs()
jobsToCreate := make(map[dax.Worker][]dax.Job)
@ -263,7 +266,7 @@ func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (internalDiffs,
return nil, errors.Wrap(err, "creating job")
}
for _, job := range jobs {
diffs.added(worker, job)
diffs.Added(worker, job)
}
}
@ -282,10 +285,24 @@ func (b *Balancer) RemoveJob(ctx context.Context, job fmt.Stringer) ([]dax.Worke
return nil, errors.Wrapf(err, "removing job: %s", job)
}
return diff.output(), nil
return diff.Output(), nil
}
func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (internalDiffs, error) {
func (b *Balancer) RemoveJobs(ctx context.Context, prefix string) ([]dax.WorkerDiff, error) {
b.mu.Lock()
defer b.mu.Unlock()
idiffs, err := b.current.DeleteJobs(ctx, b.name, prefix)
if err != nil {
return nil, errors.Wrap(err, "deleting worker jobs")
}
if err := b.freeJobs.DeleteFreeJobs(ctx, b.name, prefix); err != nil {
return nil, errors.Wrap(err, "deleting free jobs")
}
return idiffs.Output(), nil
}
func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (InternalDiffs, error) {
if worker, ok, err := b.workerForJob(ctx, job); err != nil {
return nil, errors.Wrapf(err, "getting worker for job: %s", job)
} else if ok {
@ -293,8 +310,8 @@ func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (internalDiffs, e
return nil, errors.Wrapf(err, "deleting job: %s", job)
}
diffs := newInternalDiffs()
diffs.removed(worker, job)
diffs := NewInternalDiffs()
diffs.Removed(worker, job)
return diffs, nil
}
@ -307,7 +324,7 @@ func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (internalDiffs, e
return nil, errors.Wrapf(err, "deleting free job: %s", job)
}
return internalDiffs{}, nil
return InternalDiffs{}, nil
}
// Balance ensures that all jobs are being handled by a worker by assigning jobs
@ -336,7 +353,7 @@ func (b *Balancer) Balance(ctx context.Context) ([]dax.WorkerDiff, error) {
return nil, errors.Wrap(err, "balancing jobs")
}
return diff.output(), nil
return diff.Output(), nil
}
// balance moves jobs among workers with the goal of having an equal number of
@ -346,7 +363,7 @@ func (b *Balancer) Balance(ctx context.Context) ([]dax.WorkerDiff, error) {
// the internalDiffs.merge() method, but we would need to modify that method to
// be smarter about the order in which it applies the add/remove operations.
// Until that's in place, we'll pass in a value here.
func (b *Balancer) balance(ctx context.Context, diffs internalDiffs) (internalDiffs, error) {
func (b *Balancer) balance(ctx context.Context, diffs InternalDiffs) (InternalDiffs, error) {
numWorkers, err := b.current.WorkerCount(ctx, b.name)
if err != nil {
return nil, errors.Wrapf(err, "getting worker count: %s", b.name)
@ -405,12 +422,12 @@ func (b *Balancer) balance(ctx context.Context, diffs internalDiffs) (internalDi
if rj, err := b.removeJob(ctx, sortedJobs[i]); err != nil {
return nil, errors.Wrapf(err, "removing job: %s", sortedJobs[i])
} else {
diffs.merge(rj)
diffs.Merge(rj)
}
if aj, err := b.addJobs(ctx, sortedJobs[i]); err != nil {
return nil, errors.Wrapf(err, "adding job: %s", sortedJobs[i])
} else {
diffs.merge(aj)
diffs.Merge(aj)
}
}
}
@ -547,8 +564,8 @@ func (b *Balancer) WorkersForJobPrefix(ctx context.Context, prefix string) ([]da
}
// processFreeJobs assigns all jobs in the free list to a worker.
func (b *Balancer) processFreeJobs(ctx context.Context) (internalDiffs, error) {
diffs := newInternalDiffs()
func (b *Balancer) processFreeJobs(ctx context.Context) (InternalDiffs, error) {
diffs := NewInternalDiffs()
jobs, err := b.freeJobs.ListFreeJobs(ctx, b.name)
if err != nil {
return nil, errors.Wrapf(err, "listing free jobs: %s", b.name)
@ -557,7 +574,7 @@ func (b *Balancer) processFreeJobs(ctx context.Context) (internalDiffs, error) {
if aj, err := b.addJobs(ctx, job); err != nil {
return nil, errors.Wrapf(err, "adding job: %s", job)
} else {
diffs.merge(aj)
diffs.Merge(aj)
}
if err := b.freeJobs.DeleteFreeJob(ctx, b.name, job); err != nil {
return nil, errors.Wrapf(err, "deleting free job: %s", job)

83
dax/mds/controller/naive/boltdb/balancer.go
View file

@ -289,6 +289,53 @@ func (w *workerJobService) DeleteJob(ctx context.Context, balancerName string, w
return tx.Commit()
}
func (w *workerJobService) DeleteJobs(ctx context.Context, balancerName, prefix string) (naive.InternalDiffs, error) {
tx, err := w.db.BeginTx(ctx, true)
if err != nil {
return nil, errors.Wrap(err, "beginning tx")
}
defer tx.Rollback()
bkt := tx.Bucket(bucketNaiveBalancer)
if bkt == nil {
return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer)
}
workers, err := w.getWorkers(ctx, tx, balancerName)
if err != nil {
return nil, errors.Wrap(err, "getting workers")
}
idiffs := naive.NewInternalDiffs()
for _, worker := range workers {
// get worker
wrkr := bkt.Get(workerKey(balancerName, worker))
if wrkr == nil {
panic("didn't find worker that should... definitely exist")
}
jobset, err := decodeJobSet(wrkr)
if err != nil {
return nil, errors.Wrap(err, "decoding job set")
}
jobs := jobset.RemovePrefix(prefix)
for _, job := range jobs {
idiffs.Removed(worker, job)
}
val, err := encodeJobSet(jobset)
if err != nil {
return nil, errors.Wrap(err, "encoding job set")
}
if err := bkt.Put(workerKey(balancerName, worker), val); err != nil {
return nil, errors.Wrap(err, "putting worker")
}
}
return idiffs, tx.Commit()
}
func (w *workerJobService) ListJobs(ctx context.Context, balancerName string, worker dax.Worker) (dax.Jobs, error) {
tx, err := w.db.BeginTx(ctx, false)
if err != nil {
@ -429,6 +476,42 @@ func (f *freeJobService) DeleteFreeJob(ctx context.Context, balancerName string,
return tx.Commit()
}
func (f *freeJobService) DeleteFreeJobs(ctx context.Context, balancerName, prefix string) error {
tx, err := f.db.BeginTx(ctx, true)
if err != nil {
return errors.Wrap(err, "beginning tx")
}
defer tx.Rollback()
bkt := tx.Bucket(bucketNaiveBalancer)
if bkt == nil {
return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer)
}
// get free jobs
fjs := bkt.Get(freeJobKey(balancerName))
if fjs == nil {
return nil
}
jobset, err := decodeJobSet(fjs)
if err != nil {
return errors.Wrap(err, "decoding job set")
}
jobset.RemovePrefix(prefix)
val, err := encodeJobSet(jobset)
if err != nil {
return errors.Wrap(err, "encoding job set")
}
if err := bkt.Put(freeJobKey(balancerName), val); err != nil {
return errors.Wrap(err, "putting free job")
}
return tx.Commit()
}
func (f *freeJobService) ListFreeJobs(ctx context.Context, balancerName string) (dax.Jobs, error) {
tx, err := f.db.BeginTx(ctx, false)
if err != nil {

16
dax/mds/controller/naive/types.go
View file

@ -20,13 +20,13 @@ func newJobSetDiffs() jobSetDiffs {
}
}
type internalDiffs map[dax.Worker]jobSetDiffs
type InternalDiffs map[dax.Worker]jobSetDiffs
func newInternalDiffs() internalDiffs {
return make(internalDiffs)
func NewInternalDiffs() InternalDiffs {
return make(InternalDiffs)
}
func (d internalDiffs) added(worker dax.Worker, job dax.Job) {
func (d InternalDiffs) Added(worker dax.Worker, job dax.Job) {
if _, ok := d[worker]; !ok {
d[worker] = newJobSetDiffs()
}
@ -39,7 +39,7 @@ func (d internalDiffs) added(worker dax.Worker, job dax.Job) {
d[worker].added.Add(job)
}
func (d internalDiffs) removed(worker dax.Worker, job dax.Job) {
func (d InternalDiffs) Removed(worker dax.Worker, job dax.Job) {
if _, ok := d[worker]; !ok {
d[worker] = newJobSetDiffs()
}
@ -52,7 +52,7 @@ func (d internalDiffs) removed(worker dax.Worker, job dax.Job) {
d[worker].removed.Add(job)
}
func (d internalDiffs) merge(d2 internalDiffs) {
func (d InternalDiffs) Merge(d2 InternalDiffs) {
for k, v := range d2 {
if _, ok := d[k]; !ok {
d[k] = newJobSetDiffs()
@ -62,9 +62,9 @@ func (d internalDiffs) merge(d2 internalDiffs) {
}
}
// output converts internalDiff to []controller.WorkerDiff for external
// Output converts internalDiff to []controller.WorkerDiff for external
// consumption.
func (d internalDiffs) output() []dax.WorkerDiff {
func (d InternalDiffs) Output() []dax.WorkerDiff {
out := make([]dax.WorkerDiff, len(d))
i := 0

112
dax/mds/controller/snapping_turtle.go Normal file
View file

@ -0,0 +1,112 @@
package controller
import (
"context"
"time"
"github.com/featurebasedb/featurebase/v3/dax"
)
func (c *Controller) snappingTurtleRoutine(period time.Duration, control chan struct{}) {
if period == 0 {
return // disable automatic snapshotting
}
ticker := time.NewTicker(period)
for {
select {
case <-c.stopping:
ticker.Stop()
c.logger.Debugf("TURTLE: Stopping Snapping Turtle")
return
case <-ticker.C:
c.snapAll()
case <-control:
c.snapAll()
}
}
}
func (c *Controller) snapAll() {
c.logger.Debugf("TURTLE: snapAll")
ctx := context.Background()
computeNodes, err := c.ComputeBalancer.CurrentState(ctx)
if err != nil {
c.logger.Printf("Error getting compute balancer state for snapping turtle: %v", err)
}
// Weird nested loop for snapshotting shard data. The reason for
// this is to avoid hotspotting each node in turn and spread the
// snapshotting load across all nodes rather than snapshotting all
// jobs on one node and then moving onto the next one.
i := 0
stillWorking := true
for stillWorking {
stillWorking = false
for _, workerInfo := range computeNodes {
if len(workerInfo.Jobs) <= i {
continue
}
stillWorking = true
j, err := decodeShard(workerInfo.Jobs[i])
if err != nil {
c.logger.Printf("couldn't decode a shard out of the job: '%s', err: %v", workerInfo.Jobs[i], err)
}
c.SnapshotShardData(ctx, j.t.QualifiedTableID(), j.shardNum())
}
i++
}
// Get all tables across all orgs/dbs so we can snapshot all keyed
// fields and look up whether a table is keyed to snapshot it's
// partitions.
tables, err := c.Schemar.Tables(ctx, dax.TableQualifier{})
if err != nil {
c.logger.Printf("Couldn't get schema for snapshotting keys: %v", err)
return
}
// snapshot keyed fields
tableMap := make(map[dax.TableKey]*dax.QualifiedTable)
for _, table := range tables {
tableMap[table.Key()] = table
for _, f := range table.Fields {
if f.StringKeys() && !f.IsPrimaryKey() {
err := c.SnapshotFieldKeys(ctx, table.QualifiedID(), f.Name)
if err != nil {
c.logger.Printf("Couldn't snapshot table: %s, field: %s, error: %v", table, f.Name, err)
}
}
}
}
// Get all partition jobs from balancer and snapshot table keys
// for any partition that goes with a keyed table. Doing the same
// weird nested loop thing to avoid doing all jobs on one node
// back to back.
translateNodes, err := c.TranslateBalancer.CurrentState(ctx)
if err != nil {
c.logger.Printf("Error getting translate balancer state for snapping turtle: %v", err)
}
i = 0
stillWorking = true
for stillWorking {
stillWorking = false
for _, workerInfo := range translateNodes {
if len(workerInfo.Jobs) <= i {
continue
}
stillWorking = true
j, err := decodePartition(workerInfo.Jobs[i])
if err != nil {
table := tableMap[j.table()]
if table.StringKeys() {
c.SnapshotTableKeys(ctx, table.QualifiedID(), j.partitionNum())
}
c.logger.Printf("couldn't decode a partition out of the job: '%s', err: %v", workerInfo.Jobs[i], err)
}
}
i++
}
c.logger.Debugf("TURTLE: snapAll complete")
}

28
dax/mds/controller/stringers.go
View file

@ -13,11 +13,11 @@ import (
// used as a job in the Balancer.
type pUnit struct {
t dax.TableKey
p dax.VersionedPartition
p dax.PartitionNum
}
func (p pUnit) String() string {
return fmt.Sprintf("%s|part_%d", p.t, p.p.Num)
return fmt.Sprintf("%s|part_%d", p.t, p.p)
}
func (p pUnit) table() dax.TableKey {
@ -25,14 +25,14 @@ func (p pUnit) table() dax.TableKey {
}
func (p pUnit) partitionNum() dax.PartitionNum {
return p.p.Num
return p.p
}
func partition(t dax.TableKey, p dax.VersionedPartition) pUnit {
func partition(t dax.TableKey, p dax.PartitionNum) pUnit {
return pUnit{t, p}
}
func partitions(t dax.TableKey, p ...dax.VersionedPartition) []pUnit {
func partitions(t dax.TableKey, p ...dax.PartitionNum) []pUnit {
ret := make([]pUnit, 0, len(p))
for _, vp := range p {
ret = append(ret, pUnit{t, vp})
@ -57,10 +57,7 @@ func decodePartition(j dax.Job) (pUnit, error) {
return pUnit{
t: dax.TableKey(parts[0]),
p: dax.VersionedPartition{
Num: dax.PartitionNum(intVar),
Version: -1,
},
p: dax.PartitionNum(intVar),
}, nil
}
@ -68,11 +65,11 @@ func decodePartition(j dax.Job) (pUnit, error) {
// a job in the Balancer.
type sUnit struct {
t dax.TableKey
s dax.VersionedShard
s dax.ShardNum
}
func (s sUnit) String() string {
return fmt.Sprintf("%s|shard_%s", s.t, s.s.Num)
return fmt.Sprintf("%s|shard_%s", s.t, s.s)
}
func (s sUnit) table() dax.TableKey {
@ -80,10 +77,10 @@ func (s sUnit) table() dax.TableKey {
}
func (s sUnit) shardNum() dax.ShardNum {
return s.s.Num
return s.s
}
func shard(t dax.TableKey, s dax.VersionedShard) sUnit {
func shard(t dax.TableKey, s dax.ShardNum) sUnit {
return sUnit{t, s}
}
@ -104,9 +101,6 @@ func decodeShard(j dax.Job) (sUnit, error) {
return sUnit{
t: dax.TableKey(parts[0]),
s: dax.VersionedShard{
Num: dax.ShardNum(uint64Var),
Version: -1,
},
s: dax.ShardNum(uint64Var),
}, nil
}

19
dax/mds/controller/types.go
View file

@ -1,19 +0,0 @@
package controller
import "github.com/featurebasedb/featurebase/v3/dax"
// ComputeNode represents a compute node and the table/shards for which it is
// responsible.
type ComputeNode struct {
Address dax.Address `json:"address"`
Table dax.TableKey `json:"table"`
Shards dax.ShardNums `json:"shards"`
}
// TranslateNode represents a translate node and the table/partitions for which
// it is responsible.
type TranslateNode struct {
Address dax.Address `json:"address"`
Table dax.TableKey `json:"table"`
Partitions dax.PartitionNums `json:"partitions"`
}

5
dax/mds/http/handler.go
View file

@ -7,7 +7,6 @@ import (
"github.com/gorilla/mux"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/mds"
"github.com/featurebasedb/featurebase/v3/dax/mds/controller"
)
func Handler(mds *mds.MDS) http.Handler {
@ -613,7 +612,7 @@ type ComputeNodesRequest struct {
// provided are not included in this response. That might happen if there are
// currently no active compute nodes.
type ComputeNodesResponse struct {
ComputeNodes []controller.ComputeNode `json:"compute-nodes"`
ComputeNodes []dax.ComputeNode `json:"compute-nodes"`
}
// POST /translate-nodes
@ -660,5 +659,5 @@ type TranslateNodesRequest struct {
// that partitions provided are not included in this response. That might happen
// if there are currently no active translate nodes.
type TranslateNodesResponse struct {
TranslateNodes []controller.TranslateNode `json:"translate-nodes"`
TranslateNodes []dax.TranslateNode `json:"translate-nodes"`
}

7
dax/mds/mds.go
View file

@ -29,6 +29,8 @@ type Config struct {
// have been registered.
RegistrationBatchTimeout time.Duration `toml:"registration-batch-timeout"`
SnappingTurtleTimeout time.Duration
// Poller
PollInterval time.Duration `toml:"poll-interval"`
@ -107,6 +109,7 @@ func New(cfg Config) *MDS {
TranslateBalancer: naiveboltdb.NewBalancer("translate", controllerDB, logr),
RegistrationBatchTimeout: cfg.RegistrationBatchTimeout,
SnappingTurtleTimeout: cfg.SnappingTurtleTimeout,
StorageMethod: cfg.StorageMethod,
// just reusing this bolt for internal controller svcs
@ -462,7 +465,7 @@ func (m *MDS) DeregisterNodes(ctx context.Context, addrs ...dax.Address) error {
// ComputeNodes gets the compute nodes responsible for the table/shards
// specified in the ComputeNodeRequest.
func (m *MDS) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shardNums ...dax.ShardNum) ([]controller.ComputeNode, error) {
func (m *MDS) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shardNums ...dax.ShardNum) ([]dax.ComputeNode, error) {
if err := m.sanitizeQTID(ctx, &qtid); err != nil {
return nil, errors.Wrap(err, "sanitizing")
}
@ -476,7 +479,7 @@ func (m *MDS) DebugNodes(ctx context.Context) ([]*dax.Node, error) {
// TranslateNodes gets the translate nodes responsible for the table/partitions
// specified in the TranslateNodeRequest.
func (m *MDS) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitionNums ...dax.PartitionNum) ([]controller.TranslateNode, error) {
func (m *MDS) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitionNums ...dax.PartitionNum) ([]dax.TranslateNode, error) {
if err := m.sanitizeQTID(ctx, &qtid); err != nil {
return nil, errors.Wrap(err, "sanitizing")
}

9
dax/mds/poller/poller.go
View file

@ -133,24 +133,21 @@ func (p *Poller) pollAll() {
toRemove := []dax.Address{}
for _, addr := range addrs {
p.logger.Debugf("polling: %s", addr)
start := time.Now()
up := p.nodePoller.Poll(addr)
if !up {
p.logger.Printf("poller removing %s", addr)
toRemove = append(toRemove, addr)
}
p.logger.Debugf("done poll: %s, %s", addr, time.Since(start))
}
if len(toRemove) > 0 {
p.logger.Debugf("removing addresses: %v", toRemove)
p.logger.Debugf("POLLER: removing addresses: %v", toRemove)
start := time.Now()
err := p.addressManager.RemoveAddresses(ctx, toRemove...)
if err != nil {
p.logger.Printf("removing %s: %v", toRemove, err)
p.logger.Printf("POLLER: error removing %s: %v", toRemove, err)
}
p.logger.Debugf("remove complete: %s", time.Since(start))
p.logger.Debugf("POLLER removing %v complete: %s", toRemove, time.Since(start))
}
}

12
dax/mds/schemar/boltdb/schemar.go
View file

@ -253,7 +253,7 @@ func (s *Schemar) tableIDByName(tx *boltdb.Tx, qual dax.TableQualifier, name dax
}
// Tables returns a list of Table for all existing tables. If one or more table
// names is provided, then only those will be included in the output.
// IDs is provided, then only those will be included in the output.
func (s *Schemar) Tables(ctx context.Context, qual dax.TableQualifier, ids ...dax.TableID) ([]*dax.QualifiedTable, error) {
tx, err := s.db.BeginTx(ctx, false)
if err != nil {
@ -276,6 +276,12 @@ func (s *Schemar) getTables(ctx context.Context, tx *boltdb.Tx, qual dax.TableQu
}
prefix := []byte(fmt.Sprintf(prefixFmtTables, qual.OrganizationID, qual.DatabaseID))
if qual.OrganizationID == "" && qual.DatabaseID == "" {
prefix = []byte(prefixTables)
} else if qual.DatabaseID == "" {
prefix = []byte(fmt.Sprintf(prefixFmtTablesOrg, qual.OrganizationID))
}
for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
if v == nil {
s.logger.Printf("nil value for key: %s", k)
@ -346,7 +352,9 @@ func (s *Schemar) DropTable(ctx context.Context, qtid dax.QualifiedTableID) erro
}
const (
prefixFmtTables = "tables/%s/%s/"
prefixTables = "tables/"
prefixFmtTablesOrg = prefixTables + "%s/"
prefixFmtTables = prefixFmtTablesOrg + "%s/"
prefixFmtTableNames = "tablenames/%s/%s/"
)

34
dax/mds/schemar/boltdb/schemar_test.go
View file

@ -143,4 +143,38 @@ func TestSchemar(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, exp, tables)
})
t.Run("GetTablesAll", func(t *testing.T) {
// get a fresh DB
db := testbolt.MustOpenDB(t)
defer testbolt.MustCloseDB(t, db)
t.Cleanup(func() {
testbolt.CleanupDB(t, db.Path())
})
// Initialize the buckets.
assert.NoError(t, db.InitializeBuckets(boltdb.SchemarBuckets...))
s := boltdb.NewSchemar(db, logger.NopLogger)
qtbl0 := daxtest.TestQualifiedTableWithID(t, qual, tableID0, tableName0, partitionN, false)
orgID2 := dax.OrganizationID("acme2")
qual2 := dax.NewTableQualifier(orgID2, dbID)
tableID2 := "3"
qtbl2 := daxtest.TestQualifiedTableWithID(t, qual2, tableID2, dax.TableName("two"), partitionN, false)
assert.NoError(t, s.CreateTable(ctx, qtbl0))
assert.NoError(t, s.CreateTable(ctx, qtbl2))
exp := []*dax.QualifiedTable{qtbl0, qtbl2}
tables, err := s.Tables(ctx, dax.TableQualifier{})
assert.NoError(t, err)
assert.Equal(t, exp, tables)
tables, err = s.Tables(ctx, dax.TableQualifier{OrganizationID: orgID2})
assert.NoError(t, err)
assert.Equal(t, []*dax.QualifiedTable{qtbl2}, tables)
})
}

7
dax/mds/schemar/schemar.go
View file

@ -13,6 +13,13 @@ type Schemar interface {
CreateField(context.Context, dax.QualifiedTableID, *dax.Field) error
DropField(context.Context, dax.QualifiedTableID, dax.FieldName) error
Table(context.Context, dax.QualifiedTableID) (*dax.QualifiedTable, error)
// Tables returns a list of tables. If the qualifiers DatabaseID
// is empty, all tables in the org will be returned. If the
// OrganizationID is empty, all tables will be returned. If both
// are populated, only tables in that databse will be returned. If
// greater than zero table IDs are passed in the third argument,
// only tables matching those IDs will be returned.
Tables(context.Context, dax.TableQualifier, ...dax.TableID) ([]*dax.QualifiedTable, error)
// TableID is a reverse-lookup method to get the TableID for a given

56
dax/node.go
View file

@ -29,6 +29,62 @@ type NodeService interface {
Nodes(context.Context) ([]*Node, error)
}
// ComputeNode represents a compute node and the table/shards for which it is
// responsible.
type ComputeNode struct {
Address Address `json:"address"`
Table TableKey `json:"table"`
Shards ShardNums `json:"shards"`
}
// TranslateNode represents a translate node and the table/partitions for which
// it is responsible.
type TranslateNode struct {
Address Address `json:"address"`
Table TableKey `json:"table"`
Partitions PartitionNums `json:"partitions"`
}
type Noder interface {
ComputeNodes(ctx context.Context, qtid QualifiedTableID, shards ...ShardNum) ([]ComputeNode, error)
TranslateNodes(ctx context.Context, qtid QualifiedTableID, partitions ...PartitionNum) ([]TranslateNode, error)
// IngestPartition is effectively the "write" version of TranslateNodes. Its
// implementations will return the same Address that TranslateNodes would,
// but it includes the logic to create/assign the partition if it is not
// already being handled by a computer.
IngestPartition(ctx context.Context, qtid QualifiedTableID, partition PartitionNum) (Address, error)
// IngestShard is effectively the "write" version of ComputeNodes. Its
// implementations will return the same Address that ComputeNodes would, but
// it includes the logic to create/assign the shard if it is not already
// being handled by a computer.
IngestShard(ctx context.Context, qtid QualifiedTableID, shard ShardNum) (Address, error)
}
// Ensure type implements interface.
var _ Noder = &nopNoder{}
// NopMDS is a no-op implementation of the MDS interface.
type nopNoder struct{}
func NewNopNoder() *nopNoder {
return &nopNoder{}
}
func (n *nopNoder) ComputeNodes(ctx context.Context, qtid QualifiedTableID, shards ...ShardNum) ([]ComputeNode, error) {
return nil, nil
}
func (n *nopNoder) IngestPartition(ctx context.Context, qtid QualifiedTableID, partition PartitionNum) (Address, error) {
return "", nil
}
func (n *nopNoder) IngestShard(ctx context.Context, qtid QualifiedTableID, shard ShardNum) (Address, error) {
return "", nil
}
func (n *nopNoder) TranslateNodes(ctx context.Context, qtid QualifiedTableID, partitions ...PartitionNum) ([]TranslateNode, error) {
return nil, nil
}
////////////////////////////////////////////////////
// Errors
////////////////////////////////////////////////////

41
dax/queryer/featurebase_importer.go
View file

@ -1,41 +0,0 @@
package queryer
import (
"context"
featurebase "github.com/featurebasedb/featurebase/v3"
)
// Ensure type implements interface.
var _ Importer = &FeatureBaseImporter{}
// FeatureBaseImporter is an implementation of the Importer interface which uses
// a pointer to a featurebase.API to make the underlying calls. This assumes
// those calls need to be Qcx aware, so this takes that into account.
type FeatureBaseImporter struct {
api *featurebase.API
}
func NewFeatureBaseImporter(api *featurebase.API) *FeatureBaseImporter {
return &FeatureBaseImporter{
api: api,
}
}
func (fi *FeatureBaseImporter) CreateIndexKeys(ctx context.Context, index string, keys ...string) (map[string]uint64, error) {
return fi.api.CreateIndexKeys(ctx, index, keys...)
}
func (fi *FeatureBaseImporter) CreateFieldKeys(ctx context.Context, index, field string, keys ...string) (map[string]uint64, error) {
return fi.api.CreateFieldKeys(ctx, index, field, keys...)
}
func (fi *FeatureBaseImporter) Import(ctx context.Context, req *featurebase.ImportRequest, opts ...featurebase.ImportOption) error {
qcx := fi.api.Txf().NewQcx()
return fi.api.Import(ctx, qcx, req, opts...)
}
func (fi *FeatureBaseImporter) ImportValue(ctx context.Context, req *featurebase.ImportValueRequest, opts ...featurebase.ImportOption) error {
qcx := fi.api.Txf().NewQcx()
return fi.api.ImportValue(ctx, qcx, req, opts...)
}

78
dax/queryer/interfaces.go
View file

@ -1,78 +0,0 @@
package queryer
import (
"context"
featurebase "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/mds/controller"
"github.com/featurebasedb/featurebase/v3/dax/mds/schemar"
)
type MDS interface {
// Controller-related methods.
ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) ([]controller.ComputeNode, error)
IngestPartition(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum) (dax.Address, error)
IngestShard(ctx context.Context, qtid dax.QualifiedTableID, shard dax.ShardNum) (dax.Address, error)
TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.PartitionNum) ([]controller.TranslateNode, error)
// Schemar-related methods.
schemar.Schemar
}
// Ensure type implements interface.
var _ MDS = &NopMDS{}
// NopMDS is a no-op implementation of the MDS interface.
type NopMDS struct {
schemar.NopSchemar
}
func NewNopMDS() *NopMDS {
return &NopMDS{}
}
func (m *NopMDS) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) ([]controller.ComputeNode, error) {
return nil, nil
}
func (m *NopMDS) IngestPartition(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum) (dax.Address, error) {
return "", nil
}
func (m *NopMDS) IngestShard(ctx context.Context, qtid dax.QualifiedTableID, shard dax.ShardNum) (dax.Address, error) {
return "", nil
}
func (m *NopMDS) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions ...dax.PartitionNum) ([]controller.TranslateNode, error) {
return nil, nil
}
type Importer interface {
CreateIndexKeys(ctx context.Context, index string, keys ...string) (map[string]uint64, error)
CreateFieldKeys(ctx context.Context, index, field string, keys ...string) (map[string]uint64, error)
Import(ctx context.Context, req *featurebase.ImportRequest, opts ...featurebase.ImportOption) error
ImportValue(ctx context.Context, req *featurebase.ImportValueRequest, opts ...featurebase.ImportOption) error
}
// Ensure type implements interface.
var _ Importer = &NopImporter{}
// NopImporter is a no-op implementation of the Importer interface.
type NopImporter struct{}
func NewNopImporter() *NopImporter {
return &NopImporter{}
}
func (n *NopImporter) CreateIndexKeys(ctx context.Context, index string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n *NopImporter) CreateFieldKeys(ctx context.Context, index, field string, keys ...string) (map[string]uint64, error) {
return nil, nil
}
func (n *NopImporter) Import(ctx context.Context, req *featurebase.ImportRequest, opts ...featurebase.ImportOption) error {
return nil
}
func (n *NopImporter) ImportValue(ctx context.Context, req *featurebase.ImportValueRequest, opts ...featurebase.ImportOption) error {
return nil
}

415
dax/queryer/orchestrator.go
View file

File diff suppressed because it is too large Load diff

153
dax/queryer/queryer.go
View file

@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"strings"
"sync"
"time"
featurebase "github.com/featurebasedb/featurebase/v3"
@ -30,9 +31,13 @@ import (
// that the externally-facing Molecula API would proxy query requests to a pool
// of "Queryer" nodes, which handle incoming query requests.
type Queryer struct {
orchestrator *orchestrator
mu sync.RWMutex
orchestrators map[dax.TableQualifier]*qualifiedOrchestrator
mds MDS
fbClient *featurebase.InternalClient
noder dax.Noder
schemar dax.Schemar
logger logger.Logger
}
@ -40,9 +45,10 @@ type Queryer struct {
// New returns a new instance of Queryer.
func New(cfg Config) *Queryer {
q := &Queryer{
mds: NewNopMDS(),
orchestrator: nil,
logger: logger.NopLogger,
noder: dax.NewNopNoder(),
schemar: dax.NewNopSchemar(),
orchestrators: make(map[dax.TableQualifier]*qualifiedOrchestrator),
logger: logger.NopLogger,
}
if cfg.Logger != nil {
@ -52,8 +58,63 @@ func New(cfg Config) *Queryer {
return q
}
func (q *Queryer) SetMDS(mds MDS) error {
q.mds = mds
// Orchestrator gets (or creates) an instance of qualifiedOrchestrator based on
// the provided dax.TableQualifier.
func (q *Queryer) Orchestrator(qual dax.TableQualifier) *qualifiedOrchestrator {
// Try to get orchestrator under a read lock first.
if orch := func() *qualifiedOrchestrator {
q.mu.RLock()
defer q.mu.RUnlock()
if orch, ok := q.orchestrators[qual]; ok {
return orch
}
return nil
}(); orch != nil {
return orch
}
// Since we didn't find an orchestrator under a read lock, obtain a write
// lock and try a read/write.
q.mu.Lock()
defer q.mu.Unlock()
if orch, ok := q.orchestrators[qual]; ok {
return orch
}
sapi := newQualifiedSchemaAPI(qual, q.schemar)
orch := &orchestrator{
schema: sapi,
trans: NewMDSTranslator(q.noder, q.schemar),
topology: &MDSTopology{noder: q.noder},
// TODO(jaffee) using default http.Client probably bad... need to set some timeouts.
client: q.fbClient,
stats: stats.NopStatsClient,
logger: q.logger,
}
qorch := newQualifiedOrchestrator(orch, qual)
q.orchestrators[qual] = qorch
return qorch
}
func (q *Queryer) SetNoder(noder dax.Noder) error {
q.noder = noder
return nil
}
func (q *Queryer) SetSchemar(schemar dax.Schemar) error {
q.schemar = schemar
return nil
}
func (q *Queryer) Start() error {
if q.noder == nil {
return errors.New(errors.ErrUncoded, "queryer requires noder to be configured")
} else if q.schemar == nil {
return errors.New(errors.ErrUncoded, "queryer requires schemar to be configured")
}
// fbClient is an instance of internal client. It's used in one place in the
// orchestrator (o.client.QueryNode()), but in that case, the host is
@ -67,26 +128,8 @@ func (q *Queryer) SetMDS(mds MDS) error {
if err != nil {
return errors.Wrap(err, "setting up internal client")
}
q.fbClient = fbClient
q.orchestrator = &orchestrator{
schema: NewSchemaInfoAPI(q.mds),
trans: NewMDSTranslator(q.mds),
topology: &MDSTopology{mds: q.mds},
// TODO(jaffee) using default http.Client probably bad... need to set some timeouts.
client: fbClient,
stats: stats.NopStatsClient,
logger: q.logger,
}
return nil
}
func (q *Queryer) Start() error {
if q.mds == nil {
return errors.New(errors.ErrUncoded, "queryer requires mds to be configured")
} else if q.orchestrator == nil {
return errors.New(errors.ErrUncoded, "queryer requires orchestrator to be configured")
}
return nil
}
@ -123,13 +166,10 @@ func (q *Queryer) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql str
}
// SchemaAPI
sapi := NewQualifiedSchemaAPI(qual, q.mds)
// Orchestrator
orch := newQualifiedOrchestrator(q.orchestrator, qual, q.mds)
sapi := newQualifiedSchemaAPI(qual, q.schemar)
// Importer
imp := idkmds.NewImporter(q.mds, qual, nil)
imp := idkmds.NewImporter(q.noder, q.schemar, qual, nil)
// TODO(tlt): this obviously doesn't work; we don't have an API here. We
// need a dax-compatible implementation of the SystemAPI (or at least a
@ -138,7 +178,7 @@ func (q *Queryer) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql str
systemLayer := systemlayer.NewSystemLayer()
pl := planner.NewExecutionPlanner(orch, sapi, sysapi, systemLayer, imp, q.orchestrator.logger, sql)
pl := planner.NewExecutionPlanner(q.Orchestrator(qual), sapi, sysapi, systemLayer, imp, q.logger, sql)
planOp, err := pl.CompilePlan(ctx, st)
if err != nil {
@ -204,6 +244,28 @@ func (q *Queryer) parseAndQueryPQL(ctx context.Context, qual dax.TableQualifier,
return q.QueryPQL(ctx, qual, dax.TableName(table), query)
}
// convertIndex tries to covert any "index" specified in the call.Args map to a
// TableKeyer. Note, since the Call.CallIndex() method currently only looks for
// strings, we can't just set the value to a TableKeyer; we have to set it to
// the equivalent string and then parse it back out later. A TODO would be to
// modify Call.CallIndex() to be TableKeyer aware. I didn't do that along with
// these changes because I'm not sure if we want to introduce dax types into the
// pql package.
func (q *Queryer) convertIndex(ctx context.Context, qual dax.TableQualifier, call *featurebase_pql.Call) {
if index := call.CallIndex(); index != "" {
qtbl, err := q.schemar.TableByName(ctx, qual, dax.TableName(index))
if err != nil {
return
}
call.Args["index"] = string(qtbl.Key())
}
// Apply to children.
for _, child := range call.Children {
q.convertIndex(ctx, qual, child)
}
}
func (q *Queryer) QueryPQL(ctx context.Context, qual dax.TableQualifier, table dax.TableName, pql string) (*featurebase.WireQueryResponse, error) {
// Parse the pql into a pql.Query containing []pql.Call.
qry, err := featurebase_pql.NewParser(strings.NewReader(pql)).Parse()
@ -214,12 +276,15 @@ func (q *Queryer) QueryPQL(ctx context.Context, qual dax.TableQualifier, table d
return nil, errors.Errorf("must have exactly 1 query, but got: %+v", qry.Calls)
}
tkey, err := q.indexToQualifiedTableKey(ctx, qual, string(table))
// Replace any "index" arguments within the PQL with a TableKey.
q.convertIndex(ctx, qual, qry.Calls[0])
qtbl, err := q.schemar.TableByName(ctx, qual, dax.TableName(table))
if err != nil {
return nil, errors.Wrapf(err, "converting index to qualified table key: %s", table)
return nil, errors.Wrap(err, "converting index to qualified table")
}
results, err := q.orchestrator.Execute(ctx, string(tkey), qry, nil, &featurebase.ExecOptions{})
results, err := q.Orchestrator(qual).Execute(ctx, qtbl, qry, nil, &featurebase.ExecOptions{})
if err != nil {
return nil, errors.Wrap(err, "orchestrator.Execute")
}
@ -227,10 +292,10 @@ func (q *Queryer) QueryPQL(ctx context.Context, qual dax.TableQualifier, table d
return nil, errors.Errorf("expected single result but got %+v", results.Results)
}
return PQLResultToQueryResult(results.Results[0])
return pqlResultToQueryResult(results.Results[0])
}
func PQLResultToQueryResult(pqlResult interface{}) (*featurebase.WireQueryResponse, error) {
func pqlResultToQueryResult(pqlResult interface{}) (*featurebase.WireQueryResponse, error) {
toTabler, err := server.ToTablerWrapper(pqlResult)
if err != nil {
return nil, errors.Wrap(err, "wrapping as type ToTabler")
@ -321,17 +386,3 @@ func rowToSliceInterface(header []*fbproto.ColumnInfo, row *fbproto.Row) []inter
}
return ret
}
// TODO(tlt): this method was copied from queryer/batchImporter. Can we centralize
// this logic?
func (q *Queryer) indexToQualifiedTableKey(ctx context.Context, qual dax.TableQualifier, index string) (dax.TableKey, error) {
if strings.HasPrefix(index, dax.PrefixTable+dax.TableKeyDelimiter) {
return dax.TableKey(index), nil
}
qtid, err := q.mds.TableID(ctx, qual, dax.TableName(index))
if err != nil {
return "", errors.Wrap(err, "converting index to qualified table id")
}
return qtid.Key(), nil
}

37
dax/queryer/schema_api.go
View file

@ -5,7 +5,6 @@ import (
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/mds/schemar"
"github.com/featurebasedb/featurebase/v3/errors"
)
@ -13,39 +12,33 @@ import (
var _ pilosa.SchemaAPI = (*qualifiedSchemaAPI)(nil)
// qualifiedSchemaAPI is a wrapper around schemaAPI. It is initialized with a
// TableQualifer, and it uses this qualifer to convert between, for example,
// TableQualifier, and it uses this qualifer to convert between, for example,
// FeatureBase index name (a string) and TableKey. It requires a Schemar to do
// that lookup/conversion.
type qualifiedSchemaAPI struct {
qual dax.TableQualifier
schemar schemar.Schemar
schemar dax.Schemar
}
func NewQualifiedSchemaAPI(qual dax.TableQualifier, schemar schemar.Schemar) *qualifiedSchemaAPI {
func newQualifiedSchemaAPI(qual dax.TableQualifier, schema dax.Schemar) *qualifiedSchemaAPI {
return &qualifiedSchemaAPI{
qual: qual,
schemar: schemar,
schemar: schema,
}
}
func (s *qualifiedSchemaAPI) TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error) {
qtid, err := s.schemar.TableID(ctx, s.qual, tname)
qtbl, err := s.schemar.TableByName(ctx, s.qual, tname)
if err != nil {
return nil, errors.Wrapf(err, "getting table id: (%s) %s", s.qual, tname)
}
qtbl, err := s.schemar.Table(ctx, qtid)
if err != nil {
return nil, errors.Wrapf(err, "getting table: %s", qtid)
}
return &qtbl.Table, nil
}
func (s *qualifiedSchemaAPI) TableByID(ctx context.Context, tid dax.TableID) (*dax.Table, error) {
qtid := dax.NewQualifiedTableID(s.qual, tid)
qtbl, err := s.schemar.Table(ctx, qtid)
qtbl, err := s.schemar.TableByID(ctx, qtid)
if err != nil {
return nil, errors.Wrapf(err, "getting table: %s", qtid)
}
@ -73,28 +66,28 @@ func (s *qualifiedSchemaAPI) CreateTable(ctx context.Context, tbl *dax.Table) er
}
func (s *qualifiedSchemaAPI) CreateField(ctx context.Context, tname dax.TableName, fld *dax.Field) error {
qtid, err := s.schemar.TableID(ctx, s.qual, tname)
qtbl, err := s.schemar.TableByName(ctx, s.qual, tname)
if err != nil {
return errors.Wrapf(err, "getting table id: (%s) %s", s.qual, tname)
return errors.Wrapf(err, "getting table by name: (%s) %s", s.qual, tname)
}
return s.schemar.CreateField(ctx, qtid, fld)
return s.schemar.CreateField(ctx, qtbl.QualifiedID(), fld)
}
func (s *qualifiedSchemaAPI) DeleteTable(ctx context.Context, tname dax.TableName) error {
qtid, err := s.schemar.TableID(ctx, s.qual, tname)
qtbl, err := s.schemar.TableByName(ctx, s.qual, tname)
if err != nil {
return errors.Wrapf(err, "getting table id: (%s) %s", s.qual, tname)
return errors.Wrapf(err, "getting table by name: (%s) %s", s.qual, tname)
}
return s.schemar.DropTable(ctx, qtid)
return s.schemar.DropTable(ctx, qtbl.QualifiedID())
}
func (s *qualifiedSchemaAPI) DeleteField(ctx context.Context, tname dax.TableName, fname dax.FieldName) error {
qtid, err := s.schemar.TableID(ctx, s.qual, tname)
qtid, err := s.schemar.TableByName(ctx, s.qual, tname)
if err != nil {
return errors.Wrapf(err, "getting table id: (%s) %s", s.qual, tname)
return errors.Wrapf(err, "getting table by name: (%s) %s", s.qual, tname)
}
return s.schemar.DropField(ctx, qtid, fname)
return s.schemar.DropField(ctx, qtid.Key().QualifiedTableID(), fname)
}

79
dax/queryer/schema_info_api.go
View file

@ -1,79 +0,0 @@
package queryer
import (
"context"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/mds/schemar"
"github.com/featurebasedb/featurebase/v3/errors"
)
// Ensure type implements interface.
var _ pilosa.SchemaInfoAPI = (*schemaInfoAPI)(nil)
type schemaInfoAPI struct {
schemar schemar.Schemar
}
func NewSchemaInfoAPI(schemar schemar.Schemar) *schemaInfoAPI {
return &schemaInfoAPI{
schemar: schemar,
}
}
func (a *schemaInfoAPI) IndexInfo(ctx context.Context, indexName string) (*pilosa.IndexInfo, error) {
qtid := dax.TableKey(indexName).QualifiedTableID()
tbl, err := a.schemar.Table(ctx, qtid)
if err != nil {
return nil, errors.Wrap(err, "getting table for indexinfo")
}
return daxTableToFeaturebaseIndexInfo(tbl, false)
}
func (a *schemaInfoAPI) FieldInfo(ctx context.Context, indexName, fieldName string) (*pilosa.FieldInfo, error) {
qtid := dax.TableKey(indexName).QualifiedTableID()
tbl, err := a.schemar.Table(ctx, qtid)
fldName := dax.FieldName(fieldName)
if err != nil {
return nil, errors.Wrap(err, "getting table for fieldinfo")
}
fld, ok := tbl.Field(dax.FieldName(fieldName))
if !ok {
return nil, dax.NewErrFieldDoesNotExist(fldName)
}
return pilosa.FieldToFieldInfo(fld), nil
}
// TODO(tlt): try to get rid of this in favor of pilosa.TableToIndexInfo.
// daxTableToFeaturebaseIndexInfo converts a dax.Table to a
// featurebase.IndexInfo. If useName is true, the IndexInfo.Name value will
// be set to the qualified table name. Otherwise it will be set to the table key.
func daxTableToFeaturebaseIndexInfo(qtbl *dax.QualifiedTable, useName bool) (*pilosa.IndexInfo, error) {
name := string(qtbl.Key())
if useName {
name = string(qtbl.Name)
}
ii := &pilosa.IndexInfo{
Name: name,
CreatedAt: 0,
Options: pilosa.IndexOptions{
Keys: qtbl.StringKeys(),
TrackExistence: true,
},
ShardWidth: pilosa.ShardWidth,
}
// fields
fields := make([]*pilosa.FieldInfo, len(qtbl.Fields))
for i := range qtbl.Fields {
fields[i] = pilosa.FieldToFieldInfo(qtbl.Fields[i])
}
ii.Fields = fields
return ii, nil
}

4
dax/queryer/service/queryer.go
View file

@ -50,6 +50,8 @@ func (q *queryerService) HTTPHandler() http.Handler {
}
func (q *queryerService) SetMDS(addr dax.Address) error {
q.queryer.SetMDS(mdsclient.New(addr, q.logger))
mdscli := mdsclient.New(addr, q.logger)
q.queryer.SetNoder(mdscli)
q.queryer.SetSchemar(mdscli)
return nil
}

46
dax/queryer/translator.go
View file

@ -14,15 +14,17 @@ import (
)
// Ensure type implements interface.
var _ Translator = (*MDSTranslator)(nil)
var _ Translator = (*mdsTranslator)(nil)
type MDSTranslator struct {
mds MDS
type mdsTranslator struct {
noder dax.Noder
schemar dax.Schemar
}
func NewMDSTranslator(mds MDS) *MDSTranslator {
return &MDSTranslator{
mds: mds,
func NewMDSTranslator(noder dax.Noder, schemar dax.Schemar) *mdsTranslator {
return &mdsTranslator{
noder: noder,
schemar: schemar,
}
}
@ -37,11 +39,11 @@ func fbClient(address dax.Address) (*featurebase_client.Client, error) {
)
}
func (m *MDSTranslator) CreateIndexKeys(ctx context.Context, table string, keys []string) (map[string]uint64, error) {
func (m *mdsTranslator) CreateIndexKeys(ctx context.Context, table string, keys []string) (map[string]uint64, error) {
tkey := dax.TableKey(table)
qtid := tkey.QualifiedTableID()
qtbl, err := m.mds.Table(ctx, qtid)
qtbl, err := m.schemar.TableByID(ctx, qtid)
if err != nil {
return nil, errors.Wrap(err, "getting table")
}
@ -53,7 +55,7 @@ func (m *MDSTranslator) CreateIndexKeys(ctx context.Context, table string, keys
out := make(map[string]uint64)
for pNum := range pMap {
address, err := m.mds.IngestPartition(ctx, qtid, pNum)
address, err := m.noder.IngestPartition(ctx, qtid, pNum)
if err != nil {
return nil, errors.Wrapf(err, "calling ingest-partition on table: %s, partition: %d", table, pNum)
}
@ -78,9 +80,9 @@ func (m *MDSTranslator) CreateIndexKeys(ctx context.Context, table string, keys
return out, nil
}
func (m *MDSTranslator) CreateFieldKeys(ctx context.Context, table string, field string, keys []string) (map[string]uint64, error) {
func (m *mdsTranslator) CreateFieldKeys(ctx context.Context, table string, field string, keys []string) (map[string]uint64, error) {
qtid := dax.TableKey(table).QualifiedTableID()
address, err := m.mds.IngestPartition(ctx, qtid, dax.PartitionNum(0))
address, err := m.noder.IngestPartition(ctx, qtid, dax.PartitionNum(0))
if err != nil {
return nil, errors.Wrapf(err, "calling ingest-partition on table: %s, partition: %d", table, dax.PartitionNum(0))
}
@ -96,11 +98,11 @@ func (m *MDSTranslator) CreateFieldKeys(ctx context.Context, table string, field
return fbClient.CreateFieldKeys(fld, keys...)
}
func (m *MDSTranslator) FindIndexKeys(ctx context.Context, table string, keys []string) (map[string]uint64, error) {
func (m *mdsTranslator) FindIndexKeys(ctx context.Context, table string, keys []string) (map[string]uint64, error) {
tkey := dax.TableKey(table)
qtid := tkey.QualifiedTableID()
qtbl, err := m.mds.Table(ctx, qtid)
qtbl, err := m.schemar.TableByID(ctx, qtid)
if err != nil {
return nil, errors.Wrap(err, "getting table")
}
@ -115,7 +117,7 @@ func (m *MDSTranslator) FindIndexKeys(ctx context.Context, table string, keys []
pNums = append(pNums, k)
}
translateNodes, err := m.mds.TranslateNodes(ctx, qtid, pNums...)
translateNodes, err := m.noder.TranslateNodes(ctx, qtid, pNums...)
if err != nil {
return nil, errors.Wrapf(err, "getting translate nodes for partitions on table: %s", table)
}
@ -149,9 +151,9 @@ func (m *MDSTranslator) FindIndexKeys(ctx context.Context, table string, keys []
return out, nil
}
func (m *MDSTranslator) FindFieldKeys(ctx context.Context, table, field string, keys []string) (map[string]uint64, error) {
func (m *mdsTranslator) FindFieldKeys(ctx context.Context, table, field string, keys []string) (map[string]uint64, error) {
qtid := dax.TableKey(table).QualifiedTableID()
address, err := m.mds.IngestPartition(ctx, qtid, dax.PartitionNum(0))
address, err := m.noder.IngestPartition(ctx, qtid, dax.PartitionNum(0))
if err != nil {
return nil, errors.Wrapf(err, "calling ingest-partition on table: %s, partition: %d", table, dax.PartitionNum(0))
}
@ -167,7 +169,7 @@ func (m *MDSTranslator) FindFieldKeys(ctx context.Context, table, field string,
return fbClient.FindFieldKeys(fld, keys...)
}
func (m *MDSTranslator) TranslateIndexIDs(ctx context.Context, index string, ids []uint64) ([]string, error) {
func (m *mdsTranslator) TranslateIndexIDs(ctx context.Context, index string, ids []uint64) ([]string, error) {
idsByPartition := splitIDsByPartition(index, ids, 1<<20) // TODO(jaffee), don't hardcode shardwidth...need to get this from index info
daxPartitions := make([]dax.PartitionNum, 0)
for partition := range idsByPartition {
@ -176,7 +178,7 @@ func (m *MDSTranslator) TranslateIndexIDs(ctx context.Context, index string, ids
qtid := dax.TableKey(index).QualifiedTableID()
nodes, err := m.mds.TranslateNodes(ctx, qtid, daxPartitions...)
nodes, err := m.noder.TranslateNodes(ctx, qtid, daxPartitions...)
if err != nil {
return nil, errors.Wrapf(err, "calling translate-nodes on table: %s, partitions: %v", index, daxPartitions)
}
@ -210,7 +212,7 @@ func (m *MDSTranslator) TranslateIndexIDs(ctx context.Context, index string, ids
return ret, nil
}
func (m *MDSTranslator) TranslateIndexIDSet(ctx context.Context, table string, ids map[uint64]struct{}) (map[uint64]string, error) {
func (m *mdsTranslator) TranslateIndexIDSet(ctx context.Context, table string, ids map[uint64]struct{}) (map[uint64]string, error) {
idList := make([]uint64, 0, len(ids))
for id := range ids {
idList = append(idList, id)
@ -227,7 +229,7 @@ func (m *MDSTranslator) TranslateIndexIDSet(ctx context.Context, table string, i
}
return ret, nil
}
func (m *MDSTranslator) TranslateFieldIDs(ctx context.Context, table, field string, ids map[uint64]struct{}) (map[uint64]string, error) {
func (m *mdsTranslator) TranslateFieldIDs(ctx context.Context, table, field string, ids map[uint64]struct{}) (map[uint64]string, error) {
idList := make([]uint64, 0, len(ids))
for id := range ids {
idList = append(idList, id)
@ -244,9 +246,9 @@ func (m *MDSTranslator) TranslateFieldIDs(ctx context.Context, table, field stri
}
return ret, nil
}
func (m *MDSTranslator) TranslateFieldListIDs(ctx context.Context, index, field string, ids []uint64) ([]string, error) {
func (m *mdsTranslator) TranslateFieldListIDs(ctx context.Context, index, field string, ids []uint64) ([]string, error) {
qtid := dax.TableKey(index).QualifiedTableID()
address, err := m.mds.IngestPartition(ctx, qtid, dax.PartitionNum(0))
address, err := m.noder.IngestPartition(ctx, qtid, dax.PartitionNum(0))
if err != nil {
return nil, errors.Wrapf(err, "calling ingest-partition on table: %s, partition: %d", index, dax.PartitionNum(0))
}

10
dax/role.go
View file

@ -33,8 +33,8 @@ var _ Role = &TranslateRole{}
// ComputeRole is a role specific to compute nodes.
type ComputeRole struct {
TableKey TableKey `json:"table-key"`
Shards VersionedShards `json:"shards"`
TableKey TableKey `json:"table-key"`
Shards ShardNums `json:"shards"`
}
// Type returns the type for ComputeRole. This is mainly to impolement the Role
@ -45,9 +45,9 @@ func (cr *ComputeRole) Type() RoleType {
// TranslateRole is a role specific to translate nodes.
type TranslateRole struct {
TableKey TableKey `json:"table-key"`
Partitions VersionedPartitions `json:"partitions"`
Fields VersionedFields `json:"fields"`
TableKey TableKey `json:"table-key"`
Partitions PartitionNums `json:"partitions"`
Fields []FieldName `json:"fields"`
}
// Type returns the type for TransteRole. This is mainly to impolement the Role

51
dax/schema.go Normal file
View file

@ -0,0 +1,51 @@
package dax
import "context"
// Schemar is similar to the pilosa.SchemaAPI interface, but it takes
// TableQualifiers into account.
type Schemar interface {
TableByName(ctx context.Context, qual TableQualifier, tname TableName) (*QualifiedTable, error)
TableByID(ctx context.Context, qtid QualifiedTableID) (*QualifiedTable, error)
Tables(ctx context.Context, qual TableQualifier, tids ...TableID) ([]*QualifiedTable, error)
CreateTable(ctx context.Context, qtbl *QualifiedTable) error
CreateField(ctx context.Context, qtid QualifiedTableID, fld *Field) error
DropTable(ctx context.Context, qtid QualifiedTableID) error
DropField(ctx context.Context, qtid QualifiedTableID, fname FieldName) error
}
//////////////////////////////////////////////
// Ensure type implements interface.
var _ Schemar = &NopSchemar{}
// NopSchemar is a no-op implementation of the Schemar interface.
type NopSchemar struct{}
func NewNopSchemar() *NopSchemar {
return &NopSchemar{}
}
func (s *NopSchemar) TableByName(context.Context, TableQualifier, TableName) (*QualifiedTable, error) {
return nil, nil
}
func (s *NopSchemar) TableByID(ctx context.Context, qtid QualifiedTableID) (*QualifiedTable, error) {
return nil, nil
}
func (s *NopSchemar) Tables(ctx context.Context, qual TableQualifier, tids ...TableID) ([]*QualifiedTable, error) {
return nil, nil
}
func (s *NopSchemar) CreateTable(ctx context.Context, qtbl *QualifiedTable) error {
return nil
}
func (s *NopSchemar) DropTable(ctx context.Context, qtid QualifiedTableID) error {
return nil
}
func (s *NopSchemar) CreateField(ctx context.Context, qtid QualifiedTableID, fld *Field) error {
return nil
}
func (s *NopSchemar) DropField(ctx context.Context, qtid QualifiedTableID, fld FieldName) error {
return nil
}

1
dax/server/config.go
View file

@ -85,6 +85,7 @@ func NewConfig() *Config {
Config: mds.Config{
RegistrationBatchTimeout: time.Second * 3,
StorageMethod: defaultStorageMethod,
SnappingTurtleTimeout: time.Second * 10,
},
},
Bind: ":" + defaultBindPort,

6
dax/server/server.go
View file

@ -163,8 +163,7 @@ func (m *Command) Close() error {
return nil
default:
eg := errgroup.Group{}
//eg.Go(m.Server.Close)
eg.Go(m.svcmgr.StopAll)
err := eg.Wait()
//_ = testhook.Closed(pilosa.NewAuditor(), m, nil)
close(m.done)
@ -284,6 +283,7 @@ func (m *Command) setupServices() error {
RegistrationBatchTimeout: m.Config.MDS.Config.RegistrationBatchTimeout,
StorageMethod: m.Config.MDS.Config.StorageMethod,
DataDir: m.Config.MDS.Config.DataDir,
SnappingTurtleTimeout: m.Config.MDS.Config.SnappingTurtleTimeout,
Logger: m.logger,
Director: controllerhttp.NewDirector(
controllerhttp.DirectorConfig{
@ -309,7 +309,7 @@ func (m *Command) setupServices() error {
var mdsAddr dax.Address
if m.Config.Queryer.Config.MDSAddress != "" {
mdsAddr = dax.Address(m.Config.Queryer.Config.MDSAddress + "/" + dax.ServicePrefixMDS)
mdsAddr = dax.Address(m.Config.Queryer.Config.MDSAddress)
} else if m.svcmgr.MDS != nil {
mdsAddr = m.svcmgr.MDS.Address()
} else {

12
dax/service_manager.go
View file

@ -82,6 +82,18 @@ func (s *ServiceManager) StartAll() error {
return nil
}
func (s *ServiceManager) StopAll() error {
for key := range s.computers {
if err := s.ComputerStop(key); err != nil {
s.Logger.Printf("stopping computer %s: %v", key, err)
}
}
if err := s.QueryerStop(); err != nil {
s.Logger.Printf("stopping queryer: %v", err)
}
return s.MDSStop()
}
// MDSStart starts the MDS service.
func (s *ServiceManager) MDSStart() error {
if s.MDS == nil {

20
dax/snapshot.go
View file

@ -3,12 +3,8 @@ package dax
type SnapshotShardDataRequest struct {
Address Address `json:"address"`
TableKey TableKey `json:"table-key"`
ShardNum ShardNum `json:"shard"`
FromVersion int `json:"from-version"`
ToVersion int `json:"to-version"`
Directive Directive `json:"directive"`
TableKey TableKey `json:"table-key"`
ShardNum ShardNum `json:"shard"`
}
type SnapshotTableKeysRequest struct {
@ -16,19 +12,11 @@ type SnapshotTableKeysRequest struct {
TableKey TableKey `json:"table-key"`
PartitionNum PartitionNum `json:"partition"`
FromVersion int `json:"from-version"`
ToVersion int `json:"to-version"`
Directive Directive `json:"directive"`
}
type SnapshotFieldKeysRequest struct {
Address Address `json:"address"`
TableKey TableKey `json:"table-key"`
Field FieldName `json:"field"`
FromVersion int `json:"from-version"`
ToVersion int `json:"to-version"`
Directive Directive `json:"directive"`
TableKey TableKey `json:"table-key"`
Field FieldName `json:"field"`
}

101
dax/snapshotter/api/openapi.yaml
View file

@ -1,101 +0,0 @@
openapi: 3.0.3
info:
title: Snapshotter
description: The alpha implementation of the Snapshotter interface.
version: 0.0.0
paths:
/snapshotter/health:
get:
summary: Health check endpoint.
description: Provides an endpoint to check the overall health of the Snapshotter service.
operationId: GetHealth
responses:
200:
description: Service is healthy.
/snapshotter/write-snapshot:
post:
summary: Write snapshot.
description: Write snapshot based on bucket/key.
operationId: PostWriteSnapshot
parameters:
- name: bucket
in: query
description: bucket containing snapshot key
required: true
schema:
type: string
- name: key
in: query
description: key identifying snapshot
required: true
schema:
type: string
- name: version
in: query
description: bucket/key version
required: true
schema:
type: integer
format: int64
requestBody:
content:
text/plain:
schema:
type: string
format: byte
responses:
200:
$ref: '#/components/responses/WriteSnapshotResponse'
/snapshotter/read-snapshot:
get:
summary: Read snapshot.
description: Read snapshot based on bucket/key.
operationId: GetReadSnapshot
parameters:
- name: bucket
in: query
description: bucket containing snapshot key
required: true
schema:
type: string
- name: key
in: query
description: key identifying snapshot
required: true
schema:
type: string
- name: version
in: query
description: bucket/key version
required: true
schema:
type: integer
format: int64
requestBody:
content:
text/plain:
schema:
type: string
format: byte
responses:
200:
description: Bytes making up the contents of the snapshot.
content:
text/plain:
schema:
type: string
format: byte
components:
responses:
WriteSnapshotResponse:
description: Placeholder response.
content:
application/json:
schema:
type: object

109
dax/snapshotter/client/client.go
View file

@ -1,109 +0,0 @@
// Package client contains an http implementation of the WriteLogger client.
package client
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"github.com/featurebasedb/featurebase/v3/dax"
snapshotterhttp "github.com/featurebasedb/featurebase/v3/dax/snapshotter/http"
"github.com/featurebasedb/featurebase/v3/errors"
)
const defaultScheme = "http"
// Snapshotter is a client for the Snapshotter API methods.
type Snapshotter struct {
address dax.Address
}
func New(address dax.Address) *Snapshotter {
return &Snapshotter{
address: address,
}
}
func (s *Snapshotter) Write(bucket string, key string, version int, rc io.ReadCloser) error {
url := fmt.Sprintf("%s/snapshotter/write-snapshot?bucket=%s&key=%s&version=%d",
s.address.WithScheme(defaultScheme),
url.QueryEscape(bucket),
url.QueryEscape(key),
version,
)
// Post the request.
resp, err := http.Post(url, "", rc)
if err != nil {
return errors.Wrap(err, "posting write-snapshot")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return errors.Errorf("status code: %d: %s", resp.StatusCode, b)
}
var wsr snapshotterhttp.WriteSnapshotResponse
if err := json.NewDecoder(resp.Body).Decode(&wsr); err != nil {
return errors.Wrap(err, "reading response body")
}
return nil
}
// WriteTo is exactly the same as Write, except that it takes an io.WriteTo
// instead of an io.ReadCloser. This needs to be cleaned up so that we're only
// using one or the other.
func (s *Snapshotter) WriteTo(bucket string, key string, version int, wrTo io.WriterTo) error {
url := fmt.Sprintf("%s/snapshotter/write-snapshot?bucket=%s&key=%s&version=%d",
s.address.WithScheme(defaultScheme),
url.QueryEscape(bucket),
url.QueryEscape(key),
version,
)
buf := &bytes.Buffer{}
if _, err := wrTo.WriteTo(buf); err != nil {
return errors.Wrap(err, "writing to buffer")
}
// Post the request.
resp, err := http.Post(url, "", buf)
if err != nil {
return errors.Wrap(err, "posting write-snapshot")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return errors.Errorf("status code: %d: %s", resp.StatusCode, b)
}
var wsr snapshotterhttp.WriteSnapshotResponse
if err := json.NewDecoder(resp.Body).Decode(&wsr); err != nil {
return errors.Wrap(err, "reading response body")
}
return nil
}
func (s *Snapshotter) Read(bucket string, key string, version int) (io.ReadCloser, error) {
url := fmt.Sprintf("%s/snapshotter/read-snapshot?bucket=%s&key=%s&version=%d",
s.address.WithScheme(defaultScheme),
url.QueryEscape(bucket),
url.QueryEscape(key),
version,
)
// Get the request.
resp, err := http.Get(url)
if err != nil {
return nil, errors.Wrap(err, "getting read-snapshot")
}
return resp.Body, nil
}

120
dax/snapshotter/http/handler.go
View file

@ -1,120 +0,0 @@
package http
import (
"encoding/json"
"io"
"net/http"
"strconv"
"github.com/gorilla/mux"
"github.com/featurebasedb/featurebase/v3/dax/snapshotter"
"github.com/featurebasedb/featurebase/v3/rbf"
)
func Handler(s *snapshotter.Snapshotter) http.Handler {
svr := &server{
snapshotter: s,
}
router := mux.NewRouter()
router.HandleFunc("/health", svr.getHealth).Methods("GET").Name("GetHealth")
router.HandleFunc("/write-snapshot", svr.postWriteSnapshot).Methods("POST").Name("PostWriteSnapshot")
router.HandleFunc("/read-snapshot", svr.getReadSnapshot).Methods("GET").Name("GetReadSnapshot")
return router
}
type server struct {
snapshotter *snapshotter.Snapshotter
}
// GET /health
func (s *server) getHealth(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
// POST /write-snapshot
func (s *server) postWriteSnapshot(w http.ResponseWriter, r *http.Request) {
bucket := r.URL.Query().Get("bucket")
if bucket == "" {
http.Error(w, "bucket required", http.StatusBadRequest)
return
}
key := r.URL.Query().Get("key")
if key == "" {
http.Error(w, "key required", http.StatusBadRequest)
return
}
versionArg := r.URL.Query().Get("version")
versionInt64, err := strconv.ParseInt(versionArg, 10, 64)
if err != nil {
http.Error(w, "bad shard", http.StatusBadRequest)
return
}
version := int(versionInt64)
body := r.Body
defer body.Close()
if err := s.snapshotter.Write(bucket, key, version, body); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
resp := &WriteSnapshotResponse{}
if err := json.NewEncoder(w).Encode(resp); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
type WriteSnapshotResponse struct{}
// GET /read-snapshot
func (s *server) getReadSnapshot(w http.ResponseWriter, r *http.Request) {
bucket := r.URL.Query().Get("bucket")
if bucket == "" {
http.Error(w, "bucket required", http.StatusBadRequest)
return
}
key := r.URL.Query().Get("key")
if key == "" {
http.Error(w, "key required", http.StatusBadRequest)
return
}
versionArg := r.URL.Query().Get("version")
versionInt64, err := strconv.ParseInt(versionArg, 10, 64)
if err != nil {
http.Error(w, "bad shard", http.StatusBadRequest)
return
}
version := int(versionInt64)
rc, err := s.snapshotter.Read(bucket, key, version)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
defer rc.Close()
// TODO: is rbf.PageSize a problem here for non-RBF snapshots (i.e. keys)?
// Copy data to response body.
if _, err := io.CopyBuffer(&passthroughWriter{w}, rc, make([]byte, rbf.PageSize)); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
// passthroughWriter is used to remove non-Writer interfaces from an io.Writer.
// For example, a writer that implements io.ReaderFrom can change io.Copy() behavior.
type passthroughWriter struct {
w io.Writer
}
func (w *passthroughWriter) Write(p []byte) (int, error) {
return w.w.Write(p)
}

29
dax/snapshotter/snapshotter.go
View file

@ -8,12 +8,18 @@ import (
"io/fs"
"os"
"path"
"strconv"
"sync"
"syscall"
"github.com/featurebasedb/featurebase/v3/dax/computer"
"github.com/featurebasedb/featurebase/v3/errors"
"github.com/featurebasedb/featurebase/v3/logger"
)
// bucket = table + partition or table + field
// key = "shard/num" or "keys"
type Snapshotter struct {
mu sync.RWMutex
@ -50,6 +56,29 @@ func (s *Snapshotter) Write(bucket string, key string, version int, rc io.ReadCl
return snapshotFile.Sync()
}
func (s *Snapshotter) List(bucket, key string) ([]computer.SnapInfo, error) {
dirpath := path.Join(s.dataDir, bucket, key)
entries, err := os.ReadDir(dirpath)
if err != nil {
if pe, ok := err.(*os.PathError); ok && pe.Err == syscall.ENOENT {
return nil, nil
}
return nil, errors.Wrap(err, "reading directory")
}
snaps := make([]computer.SnapInfo, len(entries))
for i, entry := range entries {
version, err := strconv.ParseInt(entry.Name(), 10, 64)
if err != nil {
return nil, errors.Wrapf(err, "filename '%s' could not be parsed to version number", entry.Name())
}
snaps[i] = computer.SnapInfo{
Version: int(version),
}
}
return snaps, nil
}
func (s *Snapshotter) Read(bucket string, key string, version int) (io.ReadCloser, error) {
_, filePath := s.paths(fullKey(bucket, key, version))
f, err := os.Open(filePath)

149
dax/storage/encoding.go Normal file
View file

@ -0,0 +1,149 @@
package storage
import (
"bufio"
"encoding/json"
"io"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/computer"
)
// TODO this needs to be genericized and moved
type TableKeyReader struct {
table dax.TableKey
partition dax.PartitionNum
scanner *bufio.Scanner
closer io.Closer
}
func NewTableKeyReader(qtid dax.QualifiedTableID, partition dax.PartitionNum, writelog io.ReadCloser) *TableKeyReader {
r := &TableKeyReader{
table: qtid.Key(),
partition: partition,
scanner: bufio.NewScanner(writelog),
closer: writelog,
}
return r
}
func (r *TableKeyReader) Read() (computer.PartitionKeyMap, error) {
if r.scanner == nil {
return computer.PartitionKeyMap{}, io.EOF
}
var b []byte
var out computer.PartitionKeyMap
if r.scanner.Scan() {
b = r.scanner.Bytes()
if err := json.Unmarshal(b, &out); err != nil {
return out, err
}
return out, nil
}
if err := r.scanner.Err(); err != nil {
return out, err
}
return out, io.EOF
}
func (r *TableKeyReader) Close() error {
if r.closer != nil {
return r.closer.Close()
}
return nil
}
type FieldKeyReader struct {
table dax.TableKey
field dax.FieldName
scanner *bufio.Scanner
closer io.Closer
}
func NewFieldKeyReader(qtid dax.QualifiedTableID, field dax.FieldName, writelog io.ReadCloser) *FieldKeyReader {
r := &FieldKeyReader{
table: qtid.Key(),
field: field,
scanner: bufio.NewScanner(writelog),
closer: writelog,
}
return r
}
func (r *FieldKeyReader) Read() (computer.FieldKeyMap, error) {
if r.scanner == nil {
return computer.FieldKeyMap{}, io.EOF
}
var b []byte
var out computer.FieldKeyMap
if r.scanner.Scan() {
b = r.scanner.Bytes()
if err := json.Unmarshal(b, &out); err != nil {
return out, err
}
return out, nil
}
if err := r.scanner.Err(); err != nil {
return out, err
}
return out, io.EOF
}
func (r *FieldKeyReader) Close() error {
if r.closer != nil {
return r.closer.Close()
}
return nil
}
type ShardReader struct {
table dax.TableKey
partition dax.PartitionNum
shard dax.ShardNum
version int
scanner *bufio.Scanner
closer io.Closer
}
func NewShardReader(qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum, writelog io.ReadCloser) *ShardReader {
r := &ShardReader{
table: qtid.Key(),
partition: partition,
shard: shard,
scanner: bufio.NewScanner(writelog),
closer: writelog,
}
return r
}
func (r *ShardReader) Read() (computer.LogMessage, error) {
if r.scanner == nil {
return nil, io.EOF
}
if r.scanner.Scan() {
return computer.UnmarshalLogMessage(r.scanner.Bytes())
}
if err := r.scanner.Err(); err != nil {
return nil, err
}
return nil, io.EOF
}
func (r *ShardReader) Close() error {
if r.closer != nil {
return r.closer.Close()
}
return nil
}

430
dax/storage/storage.go Normal file
View file

@ -0,0 +1,430 @@
package storage
import (
"fmt"
"io"
"path"
"strings"
"sync"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/computer"
"github.com/featurebasedb/featurebase/v3/errors"
"github.com/featurebasedb/featurebase/v3/logger"
)
// ResourceManager holds all the various Resources each of which is
// specific to a particular shard, table key partition or field, but
// all of which use the same underlying snapshotter and writelogger.
type ResourceManager struct {
Snapshotter computer.SnapshotService
WriteLogger computer.WriteLogService
Logger logger.Logger
mu sync.Mutex
shardResources map[shardK]*Resource
tableKeyResources map[tableKeyK]*Resource
fieldKeyResources map[fieldKeyK]*Resource
}
func NewResourceManager(s computer.SnapshotService, w computer.WriteLogService, l logger.Logger) *ResourceManager {
return &ResourceManager{
Snapshotter: s,
WriteLogger: w,
Logger: l,
shardResources: make(map[shardK]*Resource),
tableKeyResources: make(map[tableKeyK]*Resource),
fieldKeyResources: make(map[fieldKeyK]*Resource),
}
}
// compound map keys
type shardK struct {
qtid dax.QualifiedTableID
partition dax.PartitionNum
shard dax.ShardNum
}
type tableKeyK struct {
qtid dax.QualifiedTableID
partition dax.PartitionNum
}
type fieldKeyK struct {
qtid dax.QualifiedTableID
field dax.FieldName
}
func (mm *ResourceManager) GetShardResource(qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum) *Resource {
mm.mu.Lock()
defer mm.mu.Unlock()
key := shardK{qtid: qtid, partition: partition, shard: shard}
if m, ok := mm.shardResources[key]; ok {
return m
}
mm.shardResources[key] = (&Resource{
snapshotter: mm.Snapshotter,
writeLogger: mm.WriteLogger,
bucket: partitionBucket(qtid.Key(), partition),
key: shardKey(shard),
log: mm.Logger,
}).initialize()
return mm.shardResources[key]
}
func (mm *ResourceManager) RemoveShardResource(qtid dax.QualifiedTableID, partition dax.PartitionNum, shard dax.ShardNum) {
mm.mu.Lock()
defer mm.mu.Unlock()
key := shardK{qtid: qtid, partition: partition, shard: shard}
if m, ok := mm.shardResources[key]; ok {
err := m.Unlock()
if err != nil {
mm.Logger.Printf("unlocking shard resource during removal: %v", err)
}
delete(mm.shardResources, key)
}
}
func (mm *ResourceManager) GetTableKeyResource(qtid dax.QualifiedTableID, partition dax.PartitionNum) *Resource {
mm.mu.Lock()
defer mm.mu.Unlock()
key := tableKeyK{qtid: qtid, partition: partition}
if m, ok := mm.tableKeyResources[key]; ok {
return m
}
mm.tableKeyResources[key] = (&Resource{
snapshotter: mm.Snapshotter,
writeLogger: mm.WriteLogger,
bucket: partitionBucket(qtid.Key(), partition),
key: keysFileName,
log: mm.Logger,
}).initialize()
return mm.tableKeyResources[key]
}
func (mm *ResourceManager) RemoveTableKeyResource(qtid dax.QualifiedTableID, partition dax.PartitionNum) {
mm.mu.Lock()
defer mm.mu.Unlock()
key := tableKeyK{qtid: qtid, partition: partition}
if m, ok := mm.tableKeyResources[key]; ok {
err := m.Unlock()
if err != nil {
mm.Logger.Printf("unlocking table key resource during removal: %v", err)
}
delete(mm.tableKeyResources, key)
}
}
func (mm *ResourceManager) GetFieldKeyResource(qtid dax.QualifiedTableID, field dax.FieldName) *Resource {
mm.mu.Lock()
defer mm.mu.Unlock()
key := fieldKeyK{qtid: qtid, field: field}
if m, ok := mm.fieldKeyResources[key]; ok {
return m
}
mm.fieldKeyResources[key] = (&Resource{
snapshotter: mm.Snapshotter,
writeLogger: mm.WriteLogger,
bucket: fieldBucket(qtid.Key(), field),
key: keysFileName,
log: mm.Logger,
}).initialize()
return mm.fieldKeyResources[key]
}
func (mm *ResourceManager) RemoveFieldKeyResource(qtid dax.QualifiedTableID, field dax.FieldName) {
mm.mu.Lock()
defer mm.mu.Unlock()
key := fieldKeyK{qtid: qtid, field: field}
if m, ok := mm.fieldKeyResources[key]; ok {
err := m.Unlock()
if err != nil {
mm.Logger.Printf("unlocking field key resource during removal: %v", err)
}
delete(mm.fieldKeyResources, key)
}
}
// RemoveAll unlocks and deletes all resources held within this
// ResourceManager.
func (mm *ResourceManager) RemoveAll() error {
mm.mu.Lock()
defer mm.mu.Unlock()
errList := make([]error, 0)
for k, resource := range mm.shardResources {
err := resource.Unlock()
if err != nil && !strings.Contains(err.Error(), "resource was not locked") {
errList = append(errList, err)
}
delete(mm.shardResources, k)
}
for k, resource := range mm.tableKeyResources {
err := resource.Unlock()
if err != nil && !strings.Contains(err.Error(), "resource was not locked") {
errList = append(errList, err)
}
delete(mm.tableKeyResources, k)
}
for k, resource := range mm.fieldKeyResources {
err := resource.Unlock()
if err != nil && !strings.Contains(err.Error(), "resource was not locked") {
errList = append(errList, err)
}
delete(mm.fieldKeyResources, k)
}
if len(errList) > 0 {
return errors.Errorf("%v", errList)
}
return nil
}
// Resource wraps the snapshotter and writelogger to maintain messy
// state between calls. Resource is *not* threadsafe, care should be
// taken that concurrent calls are not made to Resource methods. The
// exception being that Snapshot and Append are safe to call
// concurrently.
type Resource struct {
snapshotter computer.SnapshotService
writeLogger computer.WriteLogService
bucket string
key string
log logger.Logger
loadWLsPastVersion int
latestWLVersion int
lastWLPos int
locked bool
dirty bool
}
func (m *Resource) initialize() *Resource {
m.loadWLsPastVersion = -2
m.latestWLVersion = -1
m.lastWLPos = -1
return m
}
// IsLocked checks to see if this particular instance of the resource
// believes it holds the lock. It does not look at the state of
// underlying storage to verify the lock.
func (m *Resource) IsLocked() bool {
return m.locked
}
// LoadLatestSnapshot finds the most recent snapshot for this resource
// and returns a ReadCloser for that snapshot data. If there is no
// snapshot for this resource it returns nil, nil.
func (m *Resource) LoadLatestSnapshot() (data io.ReadCloser, err error) {
snaps, err := m.snapshotter.List(m.bucket, m.key)
if err != nil {
return nil, errors.Wrap(err, "listing snapshots")
}
m.log.Debugf("LoadLatestSnapshot %s/%s: list: %v", m.bucket, m.key, snaps)
m.lastWLPos = 0
if len(snaps) == 0 {
m.loadWLsPastVersion = -1
return nil, nil
}
// assuming snapshots come back in sorted order
latest := snaps[len(snaps)-1]
m.loadWLsPastVersion = latest.Version
// TODO(jaffee): whatever is using the snapshot may discover that
// it is corrupted/incomplete. We don't want to separately check
// the checksum in here because then we'd have to read the whole
// snapshot twice. Need a way to catch the checksum error and tell
// Resource to mark that version as bad and remove it, then try
// LoadLatestSnapshot again.
return m.snapshotter.Read(m.bucket, m.key, latest.Version)
}
// // Potential future methods to support getting older versions. SnapInfo would have timestamp information as well.
//
// ListSnapshots() []SnapInfo
// LoadSnapshot(version int) (data io.ReadCloser, err error)
// LoadWriteLog can be called after LoadLatestSnapshot. It loads any
// writelog data which has been written since the latest
// snapshot. Subsequent calls to LoadWriteLog will only return new
// data that hasn't previously been returned from LoadWriteLog. If
// there is no writelog, it returns nil, nil.
func (m *Resource) LoadWriteLog() (data io.ReadCloser, err error) {
if m.loadWLsPastVersion == -2 {
return nil, errors.New(errors.ErrUncoded, "LoadWriteLog called in inconsistent state, can't tell what version to load from")
}
wLogs, err := m.writeLogger.List(m.bucket, m.key)
if err != nil {
return nil, errors.Wrap(err, "listing write logs")
}
m.log.Debugf("LoadWriteLog %s/%s: list: %v", m.bucket, m.key, wLogs)
versions := make([]int, 0, len(wLogs))
for _, log := range wLogs {
if log.Version > m.loadWLsPastVersion {
versions = append(versions, log.Version)
}
}
if len(versions) > 1 {
// TODO(jaffee) This can happen if there's a failure writing a
// snapshot. Need to implement a MultiReadCloser or similar
// that wraps all the latest write logs into one ReadCloser.
// It should only wrap the last one in a trackingReader.
return nil, errors.New(dax.ErrUnimplemented, "UNIMPLEMENTED: multiple write log versions ahead of latest snapshot.")
}
if len(versions) == 0 {
m.log.Debugf("LoadWriteLog: no logs after snapshot: %d on %s", m.loadWLsPastVersion, path.Join(m.bucket, m.key))
m.latestWLVersion = m.loadWLsPastVersion + 1
return nil, nil
}
if m.locked && m.latestWLVersion != versions[0] {
return nil, errors.New(errors.ErrUncoded, "write log version gone since locking")
}
m.latestWLVersion = versions[0]
m.dirty = true
r, err := m.writeLogger.LogReaderFrom(m.bucket, m.key, versions[0], m.lastWLPos)
if err != nil {
return nil, errors.Wrap(err, "getting writelog")
}
return &trackingReader{
r: r,
update: func(n int, err error) {
m.lastWLPos += n
},
}, nil
}
// Lock acquires an advisory lock for this resource which grants
// us exclusive access to write to it. The normal pattern is to
// call:
//
// 1. LoadLatestSnapshot
// 2. LoadWriteLog
// 3. Lock
// 4. LoadWriteLog
//
// The second call to LoadWriteLog is necessary in case any writes
// occurred between the last load and acquiring the lock. Once the
// lock is acquired it should not be possible for any more writes
// to occur. Lock will error if (a) we fail to acquire the lock or
// (b) the state of the snapshot store for this resource is not
// identical to what is was before the lock was acquired. Case (b)
// means that quite a lot has happened in between LoadWriteLog and
// Lock, and we should probably just die and start over.
func (m *Resource) Lock() error {
m.log.Debugf("Lock %s/%s", m.bucket, m.key)
// lock is sort of arbitrarily on the write log interface
if err := m.writeLogger.Lock(m.bucket, m.key); err != nil {
return errors.Wrap(err, "acquiring lock")
}
m.locked = true
return nil
}
// Append appends the msg to the write log. It will fail if we
// haven't properly loaded and gotten a lock for the resource
// we're writing to.
func (m *Resource) Append(msg []byte) error {
m.log.Debugf("Append %s/%s", m.bucket, m.key)
if m.latestWLVersion < 0 {
return errors.New(errors.ErrUncoded, "can't call append before loading and locking write log")
}
m.dirty = true
return m.writeLogger.AppendMessage(m.bucket, m.key, m.latestWLVersion, msg)
}
// IncrementWLVersion should be called during snapshotting with a
// write Tx held on the local resource. This ensures that any writes
// which completed prior to the snapshot are in the prior WL and any
// that complete after the snapshot are in the incremented WL. If
// there have been no writes since the latest snapshot, this returns
// false and does nothing. In this case, Snapshot should *not* be
// called.
func (m *Resource) IncrementWLVersion() (bool, error) {
if !m.dirty {
return false, nil
}
m.log.Debugf("IncrementWLVersion %s/%s", m.bucket, m.key)
m.latestWLVersion++
m.lastWLPos = -1
m.loadWLsPastVersion = -1
m.dirty = false
return true, nil
}
// Snapshot takes a ReadCloser which has the contents of the resource
// being tracked at a particular point in time and writes them to the
// Snapshot Store. Upon a successful write it will truncate any write
// logs which are now incorporated into the snapshot. Do not call
// until after calling IncrementWLVersion, and only if that method
// returns "true".
func (m *Resource) Snapshot(rc io.ReadCloser) error {
m.log.Debugf("Snapshot %s/%s", m.bucket, m.key)
// latestWLVersion has already been incremented at this point, so
// we write that version minus 1.
err := m.snapshotter.Write(m.bucket, m.key, m.latestWLVersion-1, rc)
if err != nil {
return errors.Wrap(err, "writing snapshot")
}
err = m.writeLogger.DeleteLog(m.bucket, m.key, m.latestWLVersion-1)
return errors.Wrap(err, "deleting old write log")
}
// SnapshotTo is Snapshot's ugly stepsister supporting the weirdness
// of reading from translate stores who we're hoping to off in the
// next season.
func (m *Resource) SnapshotTo(wt io.WriterTo) error {
m.log.Debugf("SnapshotTo %s/%s", m.bucket, m.key)
err := m.snapshotter.WriteTo(m.bucket, m.key, m.latestWLVersion-1, wt)
if err != nil {
return errors.Wrap(err, "writing snapshot SnapshotTo")
}
err = m.writeLogger.DeleteLog(m.bucket, m.key, m.latestWLVersion-1)
return errors.Wrap(err, "deleting old write log snapshotTo")
}
// Unlock releases the lock. This should be called if control of
// the underlying resource is being transitioned to another
// node. Ideally it's also called if the process crashes (e.g. via
// a defer), but an implementation based on filesystem locks
// should have those removed by the operating system when the
// process exits anyway.
func (m *Resource) Unlock() error {
m.log.Debugf("Unlock %s/%s", m.bucket, m.key)
if !m.locked {
return errors.New(errors.ErrUncoded, "resource was not locked")
}
if err := m.writeLogger.Unlock(m.bucket, m.key); err != nil {
return errors.Wrap(err, "unlocking")
}
m.locked = false
return nil
}
const (
keysFileName = "keys"
)
func partitionBucket(table dax.TableKey, partition dax.PartitionNum) string {
return path.Join(string(table), "partition", fmt.Sprintf("%d", partition))
}
func shardKey(shard dax.ShardNum) string {
return path.Join("shard", fmt.Sprintf("%d", shard))
}
func fieldBucket(table dax.TableKey, field dax.FieldName) string {
return path.Join(string(table), "field", string(field))
}

162
dax/storage/storage_test.go Normal file
View file

@ -0,0 +1,162 @@
package storage
import (
"bytes"
"io"
"os"
"testing"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/dax/snapshotter"
"github.com/featurebasedb/featurebase/v3/dax/writelogger"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/stretchr/testify/assert"
)
func TestResourceManager(t *testing.T) {
sdd, err := os.MkdirTemp("", "snaptest*")
assert.NoError(t, err)
wdd, err := os.MkdirTemp("", "wltest*")
assert.NoError(t, err)
defer func() {
os.RemoveAll(sdd)
os.RemoveAll(wdd)
}()
sn := snapshotter.New(snapshotter.Config{
DataDir: sdd,
})
wl := writelogger.New(writelogger.Config{
DataDir: wdd,
})
mm := NewResourceManager(sn, wl, logger.NewStandardLogger(os.Stderr))
qtid := dax.QualifiedTableID{
TableQualifier: dax.TableQualifier{
OrganizationID: dax.OrganizationID("org1"),
DatabaseID: dax.DatabaseID("db1"),
},
ID: dax.TableID("blah"),
Name: "blah",
}
var n int
var d, wld io.ReadCloser
// get a resource and perform normal startup routine on empty data
resource := mm.GetShardResource(qtid, dax.PartitionNum(1), dax.ShardNum(1))
d, err = resource.LoadLatestSnapshot()
assert.NoError(t, err)
assert.Nil(t, d)
wld, err = resource.LoadWriteLog()
assert.NoError(t, err)
assert.Nil(t, wld)
err = resource.Lock()
assert.NoError(t, err)
wld, err = resource.LoadWriteLog()
assert.NoError(t, err)
assert.Nil(t, wld)
// append some data
err = resource.Append([]byte("blahblah"))
assert.NoError(t, err)
// a new ResourceManager is necessary so we get a new Resource with
// new internal state instead of a cached Resource.
mm2 := NewResourceManager(sn, wl, logger.NewStandardLogger(os.Stderr))
// get second resource for same stuff
resource2 := mm2.GetShardResource(qtid, dax.PartitionNum(1), dax.ShardNum(1))
// load snapshot on 2nd resource (empty)
d, err = resource2.LoadLatestSnapshot()
assert.NoError(t, err)
assert.Nil(t, d)
// load WL on 2nd resource (blahblah)
wld, err = resource2.LoadWriteLog()
assert.NoError(t, err)
buf := make([]byte, 16)
n, _ = wld.Read(buf)
assert.Equal(t, 9, n)
assert.Equal(t, "blahblah\n", string(buf[:9]))
n, err = wld.Read(buf)
assert.Equal(t, 0, n)
assert.Equal(t, io.EOF, err)
// begin snapshot procedure on 1st resource
ok, err := resource.IncrementWLVersion()
assert.Equal(t, true, ok)
assert.NoError(t, err)
// do append on 1st resource mid-snapshot
err = resource.Append([]byte("blahbla2"))
assert.NoError(t, err)
// snapshot 1st resource
rc := io.NopCloser(bytes.NewBufferString("hahaha"))
err = resource.Snapshot(rc)
assert.NoError(t, err)
// append again on 1st resource
err = resource.Append([]byte("blahbla3"))
assert.NoError(t, err)
// locking 2nd resource should fail
err = resource2.Lock()
assert.NotNil(t, err)
// exit 1st resource
err = resource.Unlock()
assert.NoError(t, err)
// locking 2nd resource should succeed
err = resource2.Lock()
assert.NoError(t, err)
// loading write log should fail since there's been a snapshot
// between the last load and locking.
_, err = resource2.LoadWriteLog()
assert.NotNil(t, err)
// resource2 dies due to error loading write lock
err = resource2.Unlock()
assert.NoError(t, err)
// get third resource for same stuff
mm3 := NewResourceManager(sn, wl, logger.NewStandardLogger(os.Stderr))
resource3 := mm3.GetShardResource(qtid, dax.PartitionNum(1), dax.ShardNum(1))
// load snapshot on 3nd resource
d, err = resource3.LoadLatestSnapshot()
assert.NoError(t, err)
buf = make([]byte, 6)
n, err = d.Read(buf)
assert.Equal(t, 6, n)
assert.Equal(t, "hahaha", string(buf))
assert.Equal(t, nil, err)
// load write log on 3rd resource, get previous 2 writes
wld, err = resource3.LoadWriteLog()
assert.NoError(t, err)
buf = make([]byte, 20)
n, _ = wld.Read(buf)
assert.Equal(t, 18, n)
assert.Equal(t, "blahbla2\nblahbla3\n", string(buf[:18]))
n, err = wld.Read(buf)
assert.Equal(t, 0, n)
assert.Equal(t, io.EOF, err)
// lock 3rd resource
err = resource3.Lock()
assert.NoError(t, err)
// reload write log (should be empty)
wld, err = resource3.LoadWriteLog()
assert.NoError(t, err)
n, err = wld.Read(make([]byte, 8))
assert.Equal(t, 0, n)
assert.Equal(t, io.EOF, err)
}

24
dax/storage/util.go Normal file
View file

@ -0,0 +1,24 @@
package storage
import "io"
// trackingReader wraps a Reader and calls an custom "update" function
// whenever Read is called. Used by the storage layer to keep track of
// how much of the writelog has been read.
type trackingReader struct {
r io.Reader
update func(int, error)
}
func (tr *trackingReader) Read(p []byte) (n int, err error) {
n, err = tr.r.Read(p)
tr.update(n, err)
return n, err
}
func (tr *trackingReader) Close() error {
if closer, ok := tr.r.(io.Closer); ok {
return closer.Close()
}
return nil
}

40
dax/table.go
View file

@ -27,7 +27,7 @@ import (
// Table - base Table struct; includes a TableID and a TableName
// TableQualifier - combination of OrganizationID and DatabaseID
// QualifiedTable - TableQualifier plus a Table
// QualifiedTableID - TableQualifer plus a TableID
// QualifiedTableID - TableQualifier plus a TableID
// TableKey - a string representation of OrganizationID, DatabaseID, and
// TableID, which is safe to use as a FeatureBase index name.
//
@ -44,7 +44,7 @@ import (
//
////////////////////////////////////////////////////////////////////////////////
// TableKeyDelimiter is used to delimit the qualifer elements in the TableKey.
// TableKeyDelimiter is used to delimit the qualifier elements in the TableKey.
// While it might make more sense to use a pipe ("|") here, we instead use a
// double underscore because underscore is one of the few characters allowed by
// the FeatureBase index name restrictions, and we double it in a lame attempt
@ -103,6 +103,23 @@ type OrganizationID string
// value could be any string.
type DatabaseID string
// TableKeyer is an interface implemented by any type which can produce, and be
// represented by, a TableKey. In the case of a QualifiedTable, its TableKey
// might be something like `tbl__org__db__tableid`, while a general pilosa
// implemenation might represent a table as a basic table name `foo`.
type TableKeyer interface {
Key() TableKey
}
// StringTableKeyer is a helper type which can wrap a string, making it a
// TableKeyer. This is useful for certain calls to Execute() which take a string
// index name.
type StringTableKeyer string
func (s StringTableKeyer) Key() TableKey {
return TableKey(s)
}
// TableKey is a globally unique identifier for a table; it is effectively the
// compound key: (org, database, table). This is (hopefully) the value that will
// be used when interfacing with services which are unaware of table qualifiers.
@ -112,6 +129,8 @@ type DatabaseID string
// TableKey as the value for index.Name.
type TableKey string
func (t TableKey) Key() TableKey { return t }
// QualifiedTableID returns the QualifiedTableID based on the key. If TableKey
// can't be parsed into a valid (i.e. complete) QualifiedTableID, then blank
// values are used where necessary.
@ -164,7 +183,14 @@ type Table struct {
PartitionN int `json:"partitionN"`
Description string `json:"description,omitempty"`
Owner string `json:"owner,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
UpdatedAt int64 `json:"updatedAt,omitempty"`
UpdatedBy string `json:"updatedBy,omitempty"`
}
func (t *Table) Key() TableKey {
return TableKey(t.ID)
}
// CreateID generates a unique identifier for Table. If Table has already been
@ -198,7 +224,7 @@ func (t *Table) CreateID() (TableID, error) {
}
// NewTable returns a new instance of table with a pseudo-random ID which is
// assumed to be unique within the scope of a TableQualifer.
// assumed to be unique within the scope of a TableQualifier.
func NewTable(name TableName) *Table {
return &Table{
Name: name,
@ -289,7 +315,7 @@ func (o Tables) Len() int { return len(o) }
func (o Tables) Less(i, j int) bool { return o[i].Name < o[j].Name }
func (o Tables) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
// TableQualifierKey is the unique TableQualifer values encoded as a string. The
// TableQualifierKey is the unique TableQualifier values encoded as a string. The
// current encoding is delimited as `prefix|OrganizationID|DatabaseID` (where
// the pipe may be some other delimiter) by the TableQualifier.Key() method.
type TableQualifierKey string
@ -339,7 +365,7 @@ type TableQualifier struct {
DatabaseID DatabaseID `json:"db-id"`
}
// NewTableQualifier is a helper function used to create a TableQualifer from
// NewTableQualifier is a helper function used to create a TableQualifier from
// the provided arguments.
func NewTableQualifier(orgID OrganizationID, dbID DatabaseID) TableQualifier {
return TableQualifier{
@ -433,7 +459,7 @@ func (qtid QualifiedTableID) Key() TableKey {
}
// Equals returns true if `other` is the same as qtid. Note: the `Name` value is
// ignored in this comparison; only `TableQaulifer` and `ID` are considered.
// ignored in this comparison; only `TableQualifier` and `ID` are considered.
func (qtid QualifiedTableID) Equals(other QualifiedTableID) bool {
if qtid.TableQualifier == other.TableQualifier && qtid.ID == other.ID {
return true
@ -470,7 +496,7 @@ func (qt QualifiedTable) String() string {
return fmt.Sprintf("%s (%s)", qt.QualifiedID(), qt.Name)
}
// Qualifier returns the TableQualifer portion of the QualifiedTable.
// Qualifier returns the TableQualifier portion of the QualifiedTable.
func (qt *QualifiedTable) Qualifier() TableQualifier {
return qt.TableQualifier
}

17
dax/test/dax/dax_test.go
View file

@ -107,11 +107,12 @@ func TestDAXIntegration(t *testing.T) {
// skips is a list of tests which are currently not passing in dax. We
// need to get these passing before alpha.
skips := []string{
"testinsert/test-5", // error messages differ
"percentile_test/test-6", // related to TODO in orchestrator.executePercentile
"innerjointest/innerjoin-aggregate-groupby", // join test which won't work until we support multiple tables
"alterTable/alterTableBadTable", // looks like table does not exist is a different error in DAX
"top-tests/test-1", // don't know why this is failing at all
"testinsert/test-5", // error messages differ
"percentile_test/test-6", // related to TODO in orchestrator.executePercentile
"alterTable/alterTableBadTable", // looks like table does not exist is a different error in DAX
"top-tests/test-1", // don't know why this is failing at all
"delete_tests",
"subquerytable", // subqueries seem to be a problem
}
doSkip := func(name string) bool {
@ -135,12 +136,18 @@ func TestDAXIntegration(t *testing.T) {
PQLTests: make([]defs.PQLTest, 0),
}
for j, sqltest := range test.SQLTests {
if doSkip(test.Name(i)) {
continue
}
if doSkip(test.Name(i) + "/" + sqltest.Name(j)) {
continue
}
tt.SQLTests = append(tt.SQLTests, sqltest)
}
for j, pqltest := range test.PQLTests {
if doSkip(test.Name(i)) {
continue
}
if doSkip(test.Name(i) + "/" + pqltest.Name(j)) {
continue
}

30
dax/versioned_field.go
View file

@ -1,30 +0,0 @@
package dax
import "fmt"
// VersionedField is used in a similar way to VersionedShard and
// VersionedPartition in that they all contain a snapshot version.
type VersionedField struct {
Name FieldName `json:"name"`
Version int `json:"version"`
}
// String returns the VersionedField (i.e. its Name and Version) as a string.
func (f VersionedField) String() string {
return fmt.Sprintf("%s.%d", f.Name, f.Version)
}
// NewVersionedField returns a VersionedField with the provided name and version.
func NewVersionedField(name FieldName, version int) VersionedField {
return VersionedField{
Name: name,
Version: version,
}
}
// VersionedFields is a sortable slice of VersionedField.
type VersionedFields []VersionedField
func (f VersionedFields) Len() int { return len(f) }
func (f VersionedFields) Less(i, j int) bool { return f[i].Name < f[j].Name }
func (f VersionedFields) Swap(i, j int) { f[i], f[j] = f[j], f[i] }

55
dax/versioned_partition.go
View file

@ -17,55 +17,10 @@ func (p PartitionNum) String() string {
return fmt.Sprintf("%d", p)
}
// VersionedPartition is a partition number along with the snapshot version
// which it is currently writing at.
type VersionedPartition struct {
Num PartitionNum `json:"num"`
Version int `json:"version"`
}
// NewVersionedPartition returns a VersionedPartition with the provided
// partition number and version.
func NewVersionedPartition(num PartitionNum, version int) VersionedPartition {
return VersionedPartition{
Num: num,
Version: version,
func NewPartitionNums(nums ...uint64) PartitionNums {
partitions := make(PartitionNums, len(nums))
for i, n := range nums {
partitions[i] = PartitionNum(n)
}
}
// String returns the VersionedPartition (i.e. its Num and Version) as a string.
func (p VersionedPartition) String() string {
return fmt.Sprintf("%d.%d", p.Num, p.Version)
}
// VersionedPartitions is a sortable slice of VersionedPartition.
type VersionedPartitions []VersionedPartition
func (p VersionedPartitions) Len() int { return len(p) }
func (p VersionedPartitions) Less(i, j int) bool { return p[i].Num < p[j].Num }
func (p VersionedPartitions) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
// NewVersionedPartitions returns the provided list of partition nums as a list
// of VersionedPartition with an invalid version (-1). This is to use for cases
// where the request should not be aware of a partition versioning.
func NewVersionedPartitions(partitionNums ...PartitionNum) VersionedPartitions {
pvs := make(VersionedPartitions, len(partitionNums))
for i := range partitionNums {
pvs[i] = VersionedPartition{
Num: partitionNums[i],
Version: -1,
}
}
return pvs
}
// Nums returns a slice of all the partition numbers in VersionedPartitions.
func (p VersionedPartitions) Nums() []PartitionNum {
pp := make([]PartitionNum, len(p))
for i := range p {
pp[i] = p[i].Num
}
return pp
return partitions
}

56
dax/versioned_shard.go
View file

@ -12,54 +12,14 @@ func (s ShardNum) String() string {
return fmt.Sprintf("%d", s)
}
// VersionedShard is a shard number along with the snapshot version which it is
// currently writing at.
type VersionedShard struct {
Num ShardNum `json:"num"`
Version int `json:"version"`
}
func (s ShardNums) Len() int { return len(s) }
func (s ShardNums) Less(i, j int) bool { return s[i] < s[j] }
func (s ShardNums) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// NewVersionedShard returns a VersionedShard with the provided shard number and version.
func NewVersionedShard(num ShardNum, version int) VersionedShard {
return VersionedShard{
Num: num,
Version: version,
func NewShardNums(nums ...uint64) ShardNums {
shards := make(ShardNums, len(nums))
for i, n := range nums {
shards[i] = ShardNum(n)
}
}
// String returns the VersionedShard (i.e. its Num and Version) as a string.
func (s VersionedShard) String() string {
return fmt.Sprintf("%d.%d", s.Num, s.Version)
}
// VersionedShards is a sortable slice of VersionedShard.
type VersionedShards []VersionedShard
func (s VersionedShards) Len() int { return len(s) }
func (s VersionedShards) Less(i, j int) bool { return s[i].Num < s[j].Num }
func (s VersionedShards) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// NewVersionedShards returns the provided list of shard nums as a list of
// VersionedShard with an invalid version (-1). This is to use for cases where
// the request should not be aware of shard versioning.
func NewVersionedShards(shardNums ...ShardNum) VersionedShards {
svs := make(VersionedShards, len(shardNums))
for i := range shardNums {
svs[i] = VersionedShard{
Num: shardNums[i],
Version: -1,
}
}
return svs
}
// Nums returns a slice of all the shard numbers in VersionedShards.
func (s VersionedShards) Nums() []ShardNum {
ss := make([]ShardNum, len(s))
for i := range s {
ss[i] = s[i].Num
}
return ss
return shards
}

110
dax/versionstore.go
View file

@ -1,110 +0,0 @@
package dax
import (
"context"
)
// VersionStore is an interface for tracking Shard, Partition, and Field[Key]
// versions. For example, when the contents of a shard are checkpointed, and a
// snapshot is generated, and the write log messages for that shard are
// truncated, the ShardVersion for that shard is incremented. The VersionStore
// is the interface through which various services read/write that version.
type VersionStore interface {
AddTable(ctx context.Context, qtid QualifiedTableID) error
RemoveTable(ctx context.Context, qtid QualifiedTableID) (VersionedShards, VersionedPartitions, error)
// Shards (shardData)
AddShards(ctx context.Context, qtid QualifiedTableID, shards ...VersionedShard) error
Shards(ctx context.Context, qtid QualifiedTableID) (VersionedShards, bool, error)
ShardVersion(ctx context.Context, qtid QualifiedTableID, shardNum ShardNum) (int, bool, error)
ShardTables(ctx context.Context, qual TableQualifier) (TableIDs, error)
// Partitions (tableKeys)
AddPartitions(ctx context.Context, qtid QualifiedTableID, partitions ...VersionedPartition) error
Partitions(ctx context.Context, qtid QualifiedTableID) (VersionedPartitions, bool, error)
PartitionVersion(ctx context.Context, qtid QualifiedTableID, partitionNum PartitionNum) (int, bool, error)
PartitionTables(ctx context.Context, qual TableQualifier) (TableIDs, error)
// Fields (fieldKeys)
AddFields(ctx context.Context, qtid QualifiedTableID, fields ...VersionedField) error
Fields(ctx context.Context, qtid QualifiedTableID) (VersionedFields, bool, error)
FieldVersion(ctx context.Context, qtid QualifiedTableID, field FieldName) (int, bool, error)
FieldTables(ctx context.Context, qual TableQualifier) (TableIDs, error)
Copy(ctx context.Context) (VersionStore, error)
}
type DirectiveVersion interface {
Increment(ctx context.Context, delta uint64) (uint64, error)
}
// Ensure type implements interface.
var _ VersionStore = (*nopVersionStore)(nil)
// nopVersionStore is a no-op implementation of the VersionStore interface.
type nopVersionStore struct{}
// NewNopVersionStore returns a new no-op instance of VersionStore.
func NewNopVersionStore() *nopVersionStore {
return &nopVersionStore{}
}
func (s *nopVersionStore) AddTable(ctx context.Context, qtid QualifiedTableID) error {
return nil
}
func (s *nopVersionStore) RemoveTable(ctx context.Context, qtid QualifiedTableID) (VersionedShards, VersionedPartitions, error) {
return nil, nil, nil
}
func (s *nopVersionStore) AddShards(ctx context.Context, qtid QualifiedTableID, shards ...VersionedShard) error {
return nil
}
func (s *nopVersionStore) Shards(ctx context.Context, qtid QualifiedTableID) (VersionedShards, bool, error) {
return nil, false, nil
}
func (s *nopVersionStore) ShardVersion(ctx context.Context, qtid QualifiedTableID, shardNum ShardNum) (int, bool, error) {
return 0, true, nil
}
func (s *nopVersionStore) ShardTables(ctx context.Context, qual TableQualifier) (TableIDs, error) {
return TableIDs{}, nil
}
func (s *nopVersionStore) AddPartitions(ctx context.Context, qtid QualifiedTableID, partitions ...VersionedPartition) error {
return nil
}
func (s *nopVersionStore) Partitions(ctx context.Context, qtid QualifiedTableID) (VersionedPartitions, bool, error) {
return nil, false, nil
}
func (s *nopVersionStore) PartitionVersion(ctx context.Context, qtid QualifiedTableID, partitionNum PartitionNum) (int, bool, error) {
return 0, true, nil
}
func (s *nopVersionStore) PartitionTables(ctx context.Context, qual TableQualifier) (TableIDs, error) {
return TableIDs{}, nil
}
func (s *nopVersionStore) AddFields(ctx context.Context, qtid QualifiedTableID, fields ...VersionedField) error {
return nil
}
func (s *nopVersionStore) Fields(ctx context.Context, qtid QualifiedTableID) (VersionedFields, bool, error) {
return nil, false, nil
}
func (s *nopVersionStore) FieldVersion(ctx context.Context, qtid QualifiedTableID, field FieldName) (int, bool, error) {
return 0, true, nil
}
func (s *nopVersionStore) FieldTables(ctx context.Context, qual TableQualifier) (TableIDs, error) {
return TableIDs{}, nil
}
func (s *nopVersionStore) Copy(ctx context.Context) (VersionStore, error) {
return nil, nil
}

22
dax/workerjob.go
View file

@ -2,8 +2,7 @@ package dax
import (
"sort"
"golang.org/x/exp/constraints"
"strings"
)
// Worker is a generic identifier used to represent a service responsible for
@ -27,7 +26,7 @@ type Job string
// Jobs is a slice of Job.
type Jobs []Job
// WorkerInfo reprents a Worker and the Jobs to which it has been assigned.
// WorkerInfo represents a Worker and the Jobs to which it has been assigned.
type WorkerInfo struct {
ID Worker
Jobs []Job
@ -78,10 +77,10 @@ func (w WorkerDiffs) Len() int { return len(w) }
func (w WorkerDiffs) Less(i, j int) bool { return w[i].WorkerID < w[j].WorkerID }
func (w WorkerDiffs) Swap(i, j int) { w[i], w[j] = w[j], w[i] }
// Set is a set of orderable items.
type Set[K constraints.Ordered] map[K]struct{}
// Set is a set of stringy items.
type Set[K ~string] map[K]struct{}
func NewSet[K constraints.Ordered](stuff ...K) Set[K] {
func NewSet[K ~string](stuff ...K) Set[K] {
s := make(map[K]struct{})
for _, thing := range stuff {
s[thing] = struct{}{}
@ -110,6 +109,17 @@ func (s Set[K]) Remove(k K) {
delete(s, k)
}
func (s Set[K]) RemovePrefix(prefix string) []K {
ret := make([]K, 0)
for k := range s {
if strings.HasPrefix(string(k), prefix) {
ret = append(ret, k)
delete(s, k)
}
}
return ret
}
// Slice returns a slice containing each member of the set in an undefined order.
func (s Set[K]) Slice() []K {
ret := make([]K, 0, len(s))

113
dax/writelogger/api/openapi.yaml
View file

@ -1,113 +0,0 @@
openapi: 3.0.3
info:
title: WriteLogger
description: The alpha implementation of the WriteLogger interface.
version: 0.0.0
paths:
/writelogger/health:
get:
summary: Health check endpoint.
description: Provides an endpoint to check the overall health of the WriteLogger service.
operationId: GetHealth
responses:
200:
description: Service is healthy.
/writelogger/append-message:
post:
summary: Append message to WriteLogger.
description: Appends a message to a versioned bucket/key.
operationId: PostAppendMessage
requestBody:
content:
application/json:
example:
bucket: example-bucket
key: unique-key
version: 4
message: SGVsbG8gV29ybGQ=
schema:
type: object
properties:
bucket:
type: string
key:
type: string
version:
type: integer
format: int64
message:
type: string
format: byte
responses:
200:
$ref: '#/components/responses/AppendMessageResponse'
/writelogger/log-reader:
post:
summary: Read log.
description: Reads an entire log (collection of messages) at bucket/key for the given version.
operationId: PostLogReader
requestBody:
content:
application/json:
example:
bucket: example-bucket
key: unique-key
version: 4
schema:
type: object
properties:
bucket:
type: string
key:
type: string
version:
type: integer
format: int64
responses:
200:
description: Bytes making up the contents of the log.
content:
text/plain:
schema:
type: string
format: byte
/writelogger/delete-log:
post:
summary: Delete log.
description: Deletes the log at bucket/key for the given version.
operationId: PostDeleteLog
requestBody:
content:
application/json:
example:
bucket: example-bucket
key: unique-key
version: 4
schema:
type: object
properties:
bucket:
type: string
key:
type: string
version:
type: integer
format: int64
responses:
200:
description: Log was deleted.
components:
responses:
AppendMessageResponse:
description: Placeholder response.
content:
application/json:
schema:
type: object

145
dax/writelogger/client/client.go
View file

@ -1,145 +0,0 @@
// Package client contains an http implementation of the WriteLogger client.
package client
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"github.com/featurebasedb/featurebase/v3/dax"
"github.com/featurebasedb/featurebase/v3/errors"
)
const defaultScheme = "http"
// WriteLogger is a client for the WriteLogger API methods.
type WriteLogger struct {
address dax.Address
}
func New(address dax.Address) *WriteLogger {
return &WriteLogger{
address: address,
}
}
func (w *WriteLogger) AppendMessage(bucket string, key string, version int, msg []byte) error {
url := fmt.Sprintf("%s/writelogger/append-message", w.address.WithScheme(defaultScheme))
req := &AppendMessageRequest{
Bucket: bucket,
Key: key,
Version: version,
Message: msg,
}
// Encode the request.
postBody, err := json.Marshal(req)
if err != nil {
return errors.Wrap(err, "marshalling post request")
}
requestBody := bytes.NewBuffer(postBody)
// Post the request.
resp, err := http.Post(url, "application/json", requestBody)
if err != nil {
return errors.Wrap(err, "posting append-message request")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return errors.Errorf("status code: %d: %s", resp.StatusCode, b)
}
var isr *AppendMessageResponse
if err := json.NewDecoder(resp.Body).Decode(&isr); err != nil {
return errors.Wrap(err, "reading response body")
}
return nil
}
type AppendMessageRequest struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
Version int `json:"version"`
Message []byte `json:"message"`
}
type AppendMessageResponse struct{}
func (w *WriteLogger) LogReader(bucket string, key string, version int) (io.Reader, io.Closer, error) {
url := fmt.Sprintf("%s/writelogger/log-reader", w.address.WithScheme(defaultScheme))
req := &LogReaderRequest{
Bucket: bucket,
Version: version,
Key: key,
}
// Encode the request.
postBody, err := json.Marshal(req)
if err != nil {
return nil, nil, errors.Wrap(err, "marshalling post request")
}
requestBody := bytes.NewBuffer(postBody)
// Post the request.
resp, err := http.Post(url, "application/json", requestBody)
if err != nil {
return nil, nil, errors.Wrap(err, "posting log-reader request")
}
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
defer resp.Body.Close()
return nil, nil, errors.Errorf("status code: %d: %s", resp.StatusCode, b)
}
return resp.Body, resp.Body, nil
}
type LogReaderRequest struct {
Bucket string `json:"bucket"`
Version int `json:"version"`
Key string `json:"key"`
}
func (w *WriteLogger) DeleteLog(bucket string, key string, version int) error {
url := fmt.Sprintf("%s/writelogger/delete-log", w.address.WithScheme(defaultScheme))
req := &DeleteLogRequest{
Bucket: bucket,
Version: version,
Key: key,
}
// Encode the request.
postBody, err := json.Marshal(req)
if err != nil {
return errors.Wrap(err, "marshalling post request")
}
requestBody := bytes.NewBuffer(postBody)
// Post the request.
resp, err := http.Post(url, "application/json", requestBody)
if err != nil {
return errors.Wrap(err, "posting log-reader request")
}
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
defer resp.Body.Close()
return errors.Errorf("status code: %d: %s", resp.StatusCode, b)
}
return nil
}
type DeleteLogRequest struct {
Bucket string `json:"bucket"`
Version int `json:"version"`
Key string `json:"key"`
}

121
dax/writelogger/http/handler.go
View file

@ -1,121 +0,0 @@
package http
import (
"encoding/json"
"io"
"net/http"
"github.com/gorilla/mux"
"github.com/featurebasedb/featurebase/v3/dax/writelogger"
"github.com/featurebasedb/featurebase/v3/logger"
)
func Handler(w *writelogger.WriteLogger, logger logger.Logger) http.Handler {
svr := &server{
writeLogger: w,
logger: logger,
}
router := mux.NewRouter()
router.HandleFunc("/health", svr.getHealth).Methods("GET").Name("GetHealth")
router.HandleFunc("/append-message", svr.postAppendMessage).Methods("POST").Name("PostAppendMessage")
router.HandleFunc("/log-reader", svr.postLogReader).Methods("POST").Name("PostLogReader")
router.HandleFunc("/delete-log", svr.postDeleteLog).Methods("POST").Name("PostDeleteLog")
return router
}
type server struct {
writeLogger *writelogger.WriteLogger
logger logger.Logger
}
// GET /health
func (s *server) getHealth(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
// POST /append-message
func (s *server) postAppendMessage(w http.ResponseWriter, r *http.Request) {
body := r.Body
defer body.Close()
req := AppendMessageRequest{}
if err := json.NewDecoder(body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
err := s.writeLogger.AppendMessage(req.Bucket, req.Key, req.Version, req.Message)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
resp := AppendMessageResponse{}
if err := json.NewEncoder(w).Encode(resp); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
type AppendMessageRequest struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
Version int `json:"version"`
Message []byte `json:"message"`
}
type AppendMessageResponse struct{}
// POST /log-reader
func (s *server) postLogReader(w http.ResponseWriter, r *http.Request) {
body := r.Body
defer body.Close()
req := LogReaderRequest{}
if err := json.NewDecoder(body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
reader, closer, err := s.writeLogger.LogReader(req.Bucket, req.Key, req.Version)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
defer closer.Close()
if _, err := io.Copy(w, reader); err != nil {
s.logger.Printf("error streaming log data: %s", err)
}
}
type LogReaderRequest struct {
Bucket string `json:"bucket"`
Version int `json:"version"`
Key string `json:"key"`
}
// POST /delete-log
func (s *server) postDeleteLog(w http.ResponseWriter, r *http.Request) {
body := r.Body
defer body.Close()
req := DeleteLogRequest{}
if err := json.NewDecoder(body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := s.writeLogger.DeleteLog(req.Bucket, req.Key, req.Version); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
type DeleteLogRequest struct {
Bucket string `json:"bucket"`
Version int `json:"version"`
Key string `json:"key"`
}

127
dax/writelogger/writelogger.go
View file

@ -7,26 +7,31 @@ import (
"io/fs"
"os"
"path"
"strconv"
"sync"
"syscall"
"github.com/featurebasedb/featurebase/v3/dax/computer"
"github.com/featurebasedb/featurebase/v3/errors"
"github.com/featurebasedb/featurebase/v3/logger"
)
type WriteLogger struct {
mu sync.RWMutex
dataDir string
dataDir string
logFiles map[string]*os.File
mu sync.RWMutex
logFiles map[string]*os.File
lockFiles map[string]*os.File
logger logger.Logger
}
func New(cfg Config) *WriteLogger {
return &WriteLogger{
dataDir: cfg.DataDir,
logFiles: make(map[string]*os.File),
logger: logger.NopLogger,
dataDir: cfg.DataDir,
logFiles: make(map[string]*os.File),
lockFiles: make(map[string]*os.File),
logger: logger.NopLogger,
}
}
@ -43,25 +48,58 @@ func (w *WriteLogger) AppendMessage(bucket string, key string, version int, mess
return errors.Wrapf(err, "getting log file by key: %s", fKey)
}
logFile.Write(append(message, "\n"...))
logFile.Sync()
return nil
_, err = logFile.Write(append(message, "\n"...))
if err != nil {
return errors.Wrapf(err, "writing to log file %s", logFile.Name())
}
err = logFile.Sync()
return errors.Wrapf(err, "syncing log file %s", logFile.Name())
}
func (w *WriteLogger) LogReader(bucket string, key string, version int) (io.Reader, io.Closer, error) {
func (w *WriteLogger) List(bucket, key string) ([]computer.WriteLogInfo, error) {
dirpath := path.Join(w.dataDir, bucket, key)
entries, err := os.ReadDir(dirpath)
if err != nil {
if pe, ok := err.(*os.PathError); ok && pe.Err == syscall.ENOENT {
return nil, nil
}
return nil, errors.Wrap(err, "reading directory")
}
wLogs := make([]computer.WriteLogInfo, len(entries))
for i, entry := range entries {
version, err := strconv.ParseInt(entry.Name(), 10, 64)
if err != nil {
return nil, errors.Wrapf(err, "writelog filename '%s' could not be parsed to version number", entry.Name())
}
wLogs[i] = computer.WriteLogInfo{
Version: int(version),
}
}
return wLogs, nil
}
func (w *WriteLogger) LogReader(bucket, key string, version int) (io.ReadCloser, error) {
return w.LogReaderFrom(bucket, key, version, 0)
}
func (w *WriteLogger) LogReaderFrom(bucket string, key string, version int, offset int) (io.ReadCloser, error) {
_, filePath := w.paths(fullKey(bucket, key, version))
f, err := os.Open(filePath)
if err != nil {
if e, ok := err.(*fs.PathError); ok {
return nil, nil, e
return nil, e
}
return nil, nil, err
return nil, err
}
if offset > 0 {
f.Seek(int64(offset), io.SeekStart)
}
w.logger.Debugf("WriteLogger LogReader file: %s", f.Name())
return f, f, nil
return f, nil
}
func (w *WriteLogger) DeleteLog(bucket string, key string, version int) error {
@ -84,6 +122,66 @@ func (w *WriteLogger) DeleteLog(bucket string, key string, version int) error {
return os.Remove(f.Name())
}
func (w *WriteLogger) lockFile(bucket, key string) (string, string) {
lockFile := path.Join(w.dataDir, bucket, fmt.Sprintf("_lock_%s", key))
return path.Dir(lockFile), lockFile
}
func (w *WriteLogger) Lock(bucket, key string) error {
lockDir, lockFile := w.lockFile(bucket, key)
if err := os.MkdirAll(lockDir, 0777); err != nil {
return errors.Wrapf(err, "lock dir %s", lockDir)
}
f, err := os.OpenFile(lockFile, os.O_CREATE|os.O_EXCL|syscall.O_NONBLOCK, 0644)
if err != nil {
return errors.Wrapf(err, "opening lock file: %s", lockFile)
}
w.mu.Lock()
defer w.mu.Unlock()
w.lockFiles[lockFile] = f
// fd, err = syscall.Open(lockFile, syscall.O_RDWR|syscall.O_CREAT, 0644)
// if err != nil {
// return 0, errors.Wrapf(err, "syscall opening %s", lockFile)
// }
// err = syscall.FcntlFlock(uintptr(fd), syscall.F_SETLK, &syscall.Flock_t{
// Type: syscall.F_WRLCK,
// })
return nil
}
func (w *WriteLogger) Unlock(bucket, key string) error {
w.mu.Lock()
defer w.mu.Unlock()
// TODO(jaffee) since the file isn't guaranteed to be removed if
// the process is killed, we should actually use flock instead of
// EXCL file creation. Problem with that is it makes testing
// tricky because file handles from the same process are able to
// acquire the flock simultaneously. Headache.
_, lockFile := w.lockFile(bucket, key)
f, ok := w.lockFiles[lockFile]
if !ok {
return errors.New(errors.ErrUncoded, "couldn't find file to unlock")
}
f.Close()
err := os.Remove(lockFile)
delete(w.lockFiles, lockFile)
// defer func() {
// err := syscall.Close(fd)
// if err != nil {
// w.logger.Printf("error closing lockfile %s", lockFile)
// }
// }()
// err := syscall.FcntlFlock(uintptr(fd), syscall.F_SETLK, &syscall.Flock_t{
// Type: syscall.F_UNLCK,
// })
return errors.Wrap(err, "removing lock file")
}
// paths takes a key and returns the full file path (including the root data
// directory) as well as the full directory path (i.e. the file path without the
// file portion).
@ -116,6 +214,7 @@ func (w *WriteLogger) logFileByKey(key string) (*os.File, error) {
if err != nil {
return nil, errors.Wrapf(err, "opening file: %s", filePath)
}
w.logFiles[key] = f
return f, nil

6
dax/writelogger/writelogger_test.go
View file

@ -50,11 +50,11 @@ func TestWriteLogger(t *testing.T) {
assert.NoError(t, err)
// Read the message.
reader, closer, err := wl.LogReader(bucket(table, partition), key, version)
readcloser, err := wl.LogReader(bucket(table, partition), key, version)
assert.NoError(t, err)
defer closer.Close()
defer readcloser.Close()
buf, err := io.ReadAll(reader)
buf, err := io.ReadAll(readcloser)
assert.NoError(t, err)
var out payload

9
executor.go
View file

@ -47,7 +47,7 @@ const (
)
type Executor interface {
Execute(context.Context, string, *pql.Query, []uint64, *ExecOptions) (QueryResponse, error)
Execute(context.Context, dax.TableKeyer, *pql.Query, []uint64, *ExecOptions) (QueryResponse, error)
}
// executor recursively executes calls in a PQL query across all shards.
@ -77,7 +77,8 @@ type executor struct {
maxMemory int64
// Temporary flag to be removed when stablized
dataframeEnabled bool
dataframeEnabled bool
datafameUseParquet bool
}
// executorOption is a functional option type for pilosa.executor
@ -178,7 +179,9 @@ func (e *executor) InitStats() {
}
// Execute executes a PQL query.
func (e *executor) Execute(ctx context.Context, index string, q *pql.Query, shards []uint64, opt *ExecOptions) (QueryResponse, error) {
func (e *executor) Execute(ctx context.Context, tableKeyer dax.TableKeyer, q *pql.Query, shards []uint64, opt *ExecOptions) (QueryResponse, error) {
index := string(tableKeyer.Key())
span, ctx := tracing.StartSpanFromContext(ctx, "executor.Execute")
span.LogKV("pql", q.String())
defer span.Finish()

78
executor_test.go
View file

@ -33,17 +33,9 @@ import (
"github.com/featurebasedb/featurebase/v3/testhook"
. "github.com/featurebasedb/featurebase/v3/vprint" // nolint:staticcheck
"github.com/google/go-cmp/cmp"
pilosa "github.com/featurebasedb/featurebase/v3"
"github.com/featurebasedb/featurebase/v3/ctl"
"github.com/featurebasedb/featurebase/v3/disco"
"github.com/featurebasedb/featurebase/v3/logger"
"github.com/featurebasedb/featurebase/v3/pql"
"github.com/featurebasedb/featurebase/v3/proto"
"github.com/featurebasedb/featurebase/v3/server"
"github.com/featurebasedb/featurebase/v3/test"
"github.com/featurebasedb/featurebase/v3/testhook"
. "github.com/featurebasedb/featurebase/v3/vprint" // nolint:staticcheck
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
)
var (
@ -5131,47 +5123,50 @@ func TestExecutor_Execute_Extract_Keyed(t *testing.T) {
`)
resp := c.Query(t, c.Idx(), `Extract(All(), Rows(set))`)
expect := []interface{}{
pilosa.ExtractedTable{
Fields: []pilosa.ExtractedTableField{
{
Name: "set",
Type: "[]uint64",
expect := pilosa.ExtractedTable{
Fields: []pilosa.ExtractedTableField{
{
Name: "set",
Type: "[]uint64",
},
},
// The order of these probably shouldn't matter, but currently depends indirectly on the
// index.
Columns: []pilosa.ExtractedTableColumn{
{
Column: pilosa.KeyOrID{Keyed: true, Key: "h"},
Rows: []interface{}{
[]uint64{
1,
2,
},
},
},
// The order of these probably shouldn't matter, but currently depends indirectly on the
// index.
Columns: []pilosa.ExtractedTableColumn{
{
Column: pilosa.KeyOrID{Keyed: true, Key: "h"},
Rows: []interface{}{
[]uint64{
1,
2,
},
{
Column: pilosa.KeyOrID{Keyed: true, Key: "xyzzy"},
Rows: []interface{}{
[]uint64{
2,
},
},
{
Column: pilosa.KeyOrID{Keyed: true, Key: "xyzzy"},
Rows: []interface{}{
[]uint64{
2,
},
},
},
{
Column: pilosa.KeyOrID{Keyed: true, Key: "plugh"},
Rows: []interface{}{
[]uint64{},
},
},
{
Column: pilosa.KeyOrID{Keyed: true, Key: "plugh"},
Rows: []interface{}{
[]uint64{},
},
},
},
}
if !reflect.DeepEqual(expect, resp.Results) {
t.Errorf("expected %v but got %v", expect, resp.Results)
if len(resp.Results) != 1 {
t.Fail()
}
res := resp.Results[0].(pilosa.ExtractedTable)
if !reflect.DeepEqual(expect.Fields, res.Fields) {
t.Errorf("expected:\n%v\nbut got:\n%v", expect, resp.Results)
}
assert.ElementsMatch(t, expect.Columns, res.Columns)
}
func TestExecutor_Execute_MaxMemory(t *testing.T) {
@ -5791,7 +5786,7 @@ func TestExecutor_Execute_Rows_Keys(t *testing.T) {
t.Fatalf("got success, expected error similar to: %+v", test.expErr)
}
rows := res.Results[0].(pilosa.RowIdentifiers)
if !reflect.DeepEqual(rows.Keys, test.exp) {
if !assert.ElementsMatch(t, rows.Keys, test.exp) {
t.Fatalf("\ngot: %+v\nexp: %+v", rows.Keys, test.exp)
} else if rows.Rows != nil {
if test.exp == nil {
@ -7428,6 +7423,7 @@ func backupCluster(t *testing.T, c *test.Cluster, index string) (backupDir strin
buf := &bytes.Buffer{}
backupLog := logger.NewStandardLogger(buf)
backupCommand := ctl.NewBackupCommand(backupLog)
backupCommand.Host = c.Nodes[len(c.Nodes)-1].URL() // don't pick node 0 so we don't always get primary (better code coverage)
backupCommand.Index = index

266
extendiblehash/extendiblehash.go Normal file
View file

@ -0,0 +1,266 @@
package extendiblehash
import (
"bytes"
"fmt"
"github.com/featurebasedb/featurebase/v3/bufferpool"
)
// ExtendibleHashTable is an extendible hash table implementation backed by a buffer
// pool
type ExtendibleHashTable struct {
directory []bufferpool.PageID
globalDepth uint
keysPerPage int
bufferPool *bufferpool.BufferPool
}
// NewExtendibleHashTable creates a new ExtendibleHashTable
func NewExtendibleHashTable(keyLength int, valueLength int, bufferPool *bufferpool.BufferPool) (*ExtendibleHashTable, error) {
bytesPerKV := keyLength + valueLength + bufferpool.PAGE_SLOT_LENGTH
keysPerPage := (bufferpool.PAGE_SIZE - bufferpool.PAGE_SLOTS_START_OFFSET) / bytesPerKV
//create the root page
page, err := bufferPool.NewPage()
if err != nil {
return nil, err
}
page.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE)
bufferPool.FlushPage(page.ID())
return &ExtendibleHashTable{
globalDepth: 0,
directory: make([]bufferpool.PageID, 1),
keysPerPage: keysPerPage,
bufferPool: bufferPool,
}, nil
}
// Get gets a key from the hash table. It returns the value, a bool set to true if the key is
// found (false if the key is not found) or an error.
func (e *ExtendibleHashTable) Get(key []byte) ([]byte, bool, error) {
pageId, err := e.getPageID(key)
if err != nil {
return []byte{}, false, err
}
page, err := e.bufferPool.FetchPage(pageId)
if err != nil {
return []byte{}, false, err
}
defer e.bufferPool.UnpinPage(page.ID())
index, found := e.findKey(page, key)
if found {
slot := page.ReadSlot(int16(index))
return slot.ValueBytes(page), true, nil
}
return []byte{}, false, nil
}
// Put puts a key/value pair into the hash table. It returns an error if one occurs.
func (e *ExtendibleHashTable) Put(key, value []byte) error {
pageID, err := e.getPageID(key)
if err != nil {
return err
}
page, err := e.bufferPool.FetchPage(pageID)
if err != nil {
return err
}
defer e.bufferPool.UnpinPage(page.ID())
full := int(page.ReadSlotCount()) >= e.keysPerPage
err = e.putKeyValue(page, key, value)
if err != nil {
return err
}
if full {
err = e.splitOnKey(page, key)
if err != nil {
return err
}
}
return nil
}
// Close cleans up the hash table after its use.
func (e *ExtendibleHashTable) Close() {
e.bufferPool.Close()
}
func (e *ExtendibleHashTable) hashFunction(k Hashable) int {
hashResult := k.Hash() & ((1 << e.globalDepth) - 1)
return int(hashResult)
}
func (e *ExtendibleHashTable) getPageID(key []byte) (bufferpool.PageID, error) {
hash := e.hashFunction(Key(key))
if hash > len(e.directory)-1 {
return 0, fmt.Errorf("hash (%d) out of the directory array bounds (%d)", hash, len(e.directory))
}
id := e.directory[hash]
return bufferpool.PageID(id), nil
}
func (e *ExtendibleHashTable) findKey(page *bufferpool.Page, key []byte) (int, bool) {
minIndex := 0
onePastMaxIndex := int(page.ReadSlotCount())
for onePastMaxIndex != minIndex {
index := (minIndex + onePastMaxIndex) / 2
s := page.ReadSlot(int16(index))
keyAtIndex := s.KeyBytes(page)
if bytes.Equal(keyAtIndex, key) {
return index, true
}
if bytes.Compare(key, keyAtIndex) < 0 {
onePastMaxIndex = index
} else {
minIndex = index + 1
}
}
return minIndex, false
}
func (e *ExtendibleHashTable) splitOnKey(page *bufferpool.Page, key []byte) error {
if uint(page.ReadLocalDepth()) == e.globalDepth {
e.directory = append(e.directory, e.directory...)
e.globalDepth++
}
// scratch page for left
p0 := e.bufferPool.ScratchPage()
p0.WritePageNumber(int32(page.ID()))
p0.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE)
// allocate new page for split
p1, err := e.bufferPool.NewPage()
if err != nil {
return err
}
defer e.bufferPool.UnpinPage(p1.ID())
p1.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE)
// update local depths
newLocalDepth := page.ReadLocalDepth() + 1
p0.WriteLocalDepth(newLocalDepth)
p1.WriteLocalDepth(newLocalDepth)
ld := page.ReadLocalDepth()
hiBit := uint64(1 << ld)
it := bufferpool.NewPageSlotIterator(page, 0)
for {
slot := it.Next()
if slot == nil {
break
}
keyBytes := slot.KeyBytes(page)
k := string(keyBytes)
h := Key(k).Hash()
if h&hiBit > 0 {
sc := p1.ReadSlotCount()
p1.WriteKeyValueInSlot(sc, keyBytes, slot.ValueBytes(page))
// update the slot count
p1.WriteSlotCount(int16(sc + 1))
} else {
sc := p0.ReadSlotCount()
p0.WriteKeyValueInSlot(sc, keyBytes, slot.ValueBytes(page))
// update the slot count
p0.WriteSlotCount(int16(sc + 1))
}
}
for j := Key(key).Hash() & (hiBit - 1); j < uint64(len(e.directory)); j += hiBit {
if j&hiBit > 0 {
e.directory[j] = p1.ID()
} else {
e.directory[j] = p0.ID()
}
}
// copy p1 back into page
p0.WritePage(page)
return nil
}
func (e *ExtendibleHashTable) cleanPage(page *bufferpool.Page) error {
scratch := e.bufferPool.ScratchPage()
// copy page number
scratch.WritePageNumber(int32(page.ID()))
// set the page type
scratch.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE)
// copy local depth
scratch.WriteLocalDepth(page.ReadLocalDepth())
// copy slots from page to scratch
si := bufferpool.NewPageSlotIterator(page, 0)
for {
slot := si.Next()
if slot == nil {
break
}
scratch.WriteKeyValueInSlot(si.Cursor(), slot.KeyBytes(page), slot.ValueBytes(page))
}
// update the slot count
scratch.WriteSlotCount(page.ReadSlotCount())
// write scratch back to page
scratch.WritePage(page)
return nil
}
func (e *ExtendibleHashTable) keyValueWillFit(page *bufferpool.Page, key, value []byte) bool {
// will this k/v fit on the page?
slotLen := 4 // we need 2 len words for the slot
chunkLen := 6 + len(key) + len(value) // int16 len + int32 len + len of respective []byte
fs := page.FreeSpace()
return fs > (int16(slotLen) + int16(chunkLen))
}
func (e *ExtendibleHashTable) putKeyValue(page *bufferpool.Page, key, value []byte) error {
if !e.keyValueWillFit(page, key, value) {
// try to garbage collect the page first
err := e.cleanPage(page)
if err != nil {
return err
}
}
//find the key
newIndex, found := e.findKey(page, []byte(key))
// get the slot count
slotCount := int(page.ReadSlotCount())
if found {
// we found the key, so we will update the value
err := page.WriteKeyValueInSlot(int16(newIndex), []byte(key), []byte(value))
if err != nil {
return err
}
} else {
// TODO(pok) should check in WriteSlot() to see if we are out space too...
// TODO(pok) we should move all the slots in one fell swoop, because,... performance
// move all the slots after where we are going to insert
for j := slotCount; j > newIndex; j-- {
sl := page.ReadSlot(int16(j - 1))
page.WriteSlot(int16(j), sl)
}
err := page.WriteKeyValueInSlot(int16(newIndex), []byte(key), []byte(value))
if err != nil {
return err
}
// update the slot count
page.WriteSlotCount(int16(slotCount + 1))
}
return nil
}

327
extendiblehash/extendiblehash_test.go Normal file
View file

@ -0,0 +1,327 @@
package extendiblehash
import (
"strconv"
"testing"
"github.com/featurebasedb/featurebase/v3/bufferpool"
"github.com/stretchr/testify/assert"
)
func makeDirectory() (*ExtendibleHashTable, error) {
diskManager := bufferpool.NewInMemDiskSpillingDiskManager(128)
bufferPool := bufferpool.NewBufferPool(128, diskManager)
keySize := 12
valueSize := 20
return NewExtendibleHashTable(keySize, valueSize, bufferPool)
}
func TestHashTable_ExtendibleHash(t *testing.T) {
d, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
d.globalDepth = 4
key := "321" // 0011
key2 := "123" // 1011
result := d.hashFunction(Key(key))
result2 := d.hashFunction(Key(key2))
assert.Equal(t, 7, result)
assert.Equal(t, 6, result2)
}
func TestHashTable_GetPage(t *testing.T) {
d, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
d.globalDepth = 4
d.directory = make([]bufferpool.PageID, 16)
key := "478"
d.directory[14] = 2
pageID, err := d.getPageID([]byte(key))
if err != nil {
t.Fatal(err)
}
assert.Equal(t, 2, int(pageID))
}
func TestHashTable_GetPage_ShouldReturnError_WhenOffsetIsNotLimitedToDataSize(t *testing.T) {
d, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
d.globalDepth = 4
key := "478"
_, err = d.getPageID([]byte(key))
assert.Error(t, err)
}
func TestHashTable_GetPage_ShouldReturnError_WhenPageIDIsOutOfTheTable(t *testing.T) {
d, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
d.directory = make([]bufferpool.PageID, 0)
key := "123"
_, err = d.getPageID([]byte(key))
assert.Error(t, err)
}
func TestHashTable_Get(t *testing.T) {
d, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
d.globalDepth = 4
d.directory = make([]bufferpool.PageID, 16)
d.directory[14] = 2
// force there to be two pages
page, err := d.bufferPool.NewPage() //1
if err != nil {
t.Fatal(err)
}
page.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE)
d.bufferPool.FlushPage(page.ID())
page, err = d.bufferPool.NewPage() //2
if err != nil {
t.Fatal(err)
}
page.WritePageType(bufferpool.PAGE_TYPE_HASH_TABLE)
d.bufferPool.FlushPage(page.ID())
// now do the test
page, err = d.bufferPool.FetchPage(2)
if err != nil {
t.Fatal(err)
}
defer d.bufferPool.UnpinPage(page.ID())
key := "478"
value := "Hi"
page.WriteKeyValueInSlot(0, []byte(key), []byte(value))
page.WriteSlotCount(int16(1))
result, _, err := d.Get([]byte(key))
if err != nil {
t.Fatal(err)
}
assert.Equal(t, "Hi", string(result))
}
func TestHashTable_Get_ShouldHandleError(t *testing.T) {
d, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
key := "123"
result, found, err := d.Get([]byte(key))
assert.Equal(t, err, nil)
assert.Equal(t, []byte{}, result)
assert.Equal(t, false, found)
}
func TestHashTable_Put(t *testing.T) {
d, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
page, err := d.bufferPool.FetchPage(0)
if err != nil {
t.Fatal(err)
}
defer d.bufferPool.UnpinPage(page.ID())
err = addToPage(page, 5)
if err != nil {
t.Fatal(err)
}
d.Put([]byte("123"), []byte("Yolo !"))
value, found, err := d.Get([]byte("123"))
if err != nil {
t.Fatal(err)
}
assert.Equal(t, "Yolo !", string(value))
assert.Equal(t, true, found)
}
func TestHashTable_Put_ShouldIncreaseSize_WhenTableIsFull(t *testing.T) {
d, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
page, err := d.bufferPool.FetchPage(0)
if err != nil {
t.Fatal(err)
}
defer d.bufferPool.UnpinPage(page.ID())
err = addToPage(page, 227) // keys per page with key 12, value 20
if err != nil {
t.Fatal(err)
}
d.Put([]byte("123"), []byte("Yolo !"))
value, _, err := d.Get([]byte("123"))
if err != nil {
t.Fatal(err)
}
assert.Equal(t, "Yolo !", string(value))
assert.Equal(t, 2, len(d.directory))
assert.Equal(t, uint(1), d.globalDepth)
}
func TestHashTable_PutShouldIncrementLD_WhenPageIsFull(t *testing.T) {
d, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
page, err := d.bufferPool.FetchPage(0)
if err != nil {
t.Fatal(err)
}
defer d.bufferPool.UnpinPage(page.ID())
err = addToPage(page, 227) // keys per page with key 12, value 20
if err != nil {
t.Fatal(err)
}
d.Put([]byte("12345678"), []byte("Yolo !"))
assert.Equal(t, int64(8192*2), d.bufferPool.OnDiskSize())
assert.Equal(t, 1, int(d.globalDepth))
p0, err := d.bufferPool.FetchPage(0)
if err != nil {
t.Fatal(err)
}
defer d.bufferPool.UnpinPage(p0.ID())
assert.Equal(t, int16(1), p0.ReadLocalDepth())
p1, err := d.bufferPool.FetchPage(1)
if err != nil {
t.Fatal(err)
}
defer d.bufferPool.UnpinPage(p1.ID())
assert.Equal(t, int16(1), p1.ReadLocalDepth())
}
func TestHashTable_Put_INT(t *testing.T) {
d, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
for i := 0; i < 4000; i++ {
err = d.Put([]byte("key"+strconv.Itoa(i)), []byte("Yolo !"))
if err != nil {
t.Fatal(err)
}
}
assert.Equal(t, []bufferpool.PageID{0, 1, 2, 3, 4, 7, 6, 5, 13, 14, 12, 9, 8, 15, 10, 11, 28, 24, 21, 18, 4, 19, 29, 20, 27, 22, 25, 23, 17, 15, 16, 26}, d.directory)
}
func TestHashTable_Put_SameKey_ALotOfTime(t *testing.T) {
d, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
for i := 0; i < 10000; i++ {
d.Put([]byte("key"), []byte("Yolo ! "+strconv.Itoa(i)))
}
value, _, err := d.Get([]byte("key"))
if err != nil {
t.Fatal(err)
}
assert.Equal(t, "Yolo ! 9999", string(value))
assert.Equal(t, 1, len(d.directory))
assert.Equal(t, int64(8192), d.bufferPool.OnDiskSize())
}
func TestHashTable_Put_Many_Keys(t *testing.T) {
d, err := makeDirectory()
if err != nil {
t.Fatal(err)
}
for i := 0; i < 1000000; i++ {
err = d.Put([]byte("key"+strconv.Itoa(i)), []byte("Yolo ! "+strconv.Itoa(i)))
if err != nil {
t.Fatal(err)
}
}
value, _, err := d.Get([]byte("key99756"))
if err != nil {
t.Fatal(err)
}
assert.Equal(t, "Yolo ! 99756", string(value))
assert.Equal(t, 8192, len(d.directory))
assert.Equal(t, uint(13), d.globalDepth)
d.Close()
}
func BenchmarkHashTable_Put_Many_Keys(b *testing.B) {
for i := 0; i < b.N; i++ {
d, err := makeDirectory()
if err != nil {
b.Fatal(err)
}
for i := 0; i < 1000000; i++ {
err = d.Put([]byte("key"+strconv.Itoa(i)), []byte("Yolo ! "+strconv.Itoa(i)))
if err != nil {
b.Fatal(err)
}
}
value, _, err := d.Get([]byte("key99756"))
if err != nil {
b.Fatal(err)
}
assert.Equal(b, "Yolo ! 99756", string(value))
assert.Equal(b, 8192, len(d.directory))
assert.Equal(b, uint(13), d.globalDepth)
d.Close()
}
}
func addToPage(page *bufferpool.Page, numberOfRecords int) error {
for i := 0; i < numberOfRecords; i++ {
//fmt.Printf("writing record %d\n", i+1)
itoa := strconv.Itoa(i)
err := page.WriteKeyValueInSlot(int16(i), []byte("key"+itoa), []byte("value foo bar"))
if err != nil {
return err
}
page.WriteSlotCount(int16(page.ReadSlotCount() + 1))
}
return nil
}

Some files were not shown because too many files have changed in this diff Show more