diff --git a/.gitignore b/.gitignore index dabd25b84..b3b471f17 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,4 @@ dax/dax-data coverage-from-docker *.client_id.txt + diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 997156a54..7276b0a45 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -255,6 +255,16 @@ run go tests race: stage: nonblocking # don't let this job block any other jobs because it takes much longer than the other tests. image: golang:$GOVERSION extends: .go-cache + variables: + SQLDB_DB: run_go_tests_race + POSTGRES_DB: run_go_tests_race + SQLDB_USER: postgres + POSTGRES_USER: postgres + SQLDB_PASSWORD: $POSTGRES_PASSWORD + POSTGRES_PASSWORD: $POSTGRES_PASSWORD + SQLDB_HOST: postgres + services: + - postgres:14.7 rules: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' needs: ["smoke build"] # we do block on smoke build though bc it's pretty dumb to test stuff if it doesn't build @@ -276,6 +286,16 @@ run go tests: stage: test image: golang:$GOVERSION extends: .go-cache + variables: + SQLDB_DB: run_go_tests + POSTGRES_DB: run_go_tests + SQLDB_USER: postgres + POSTGRES_USER: postgres + SQLDB_PASSWORD: $POSTGRES_PASSWORD + POSTGRES_PASSWORD: $POSTGRES_PASSWORD + SQLDB_HOST: postgres + services: + - postgres:14.7 rules: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: @@ -295,14 +315,29 @@ run go tests: run go tests dax/test/dax: stage: test image: golang:$GOVERSION + extends: .go-cache tags: - - aws + - docker + variables: + SQLDB_DB: run_go_tests_dax + POSTGRES_DB: run_go_tests_dax + SQLDB_USER: postgres + POSTGRES_USER: postgres + SQLDB_PASSWORD: $POSTGRES_PASSWORD + POSTGRES_PASSWORD: $POSTGRES_PASSWORD + SQLDB_HOST: postgres + services: + - postgres:14.7 rules: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - echo "Building FB and Datagen docker images for DAX tests" - PKG_LIST=$(go list ./... | grep -Ev 'internal/clustertests|simulacraData' | paste -s -d, -) + - export TMPDIR=/mnt/ramdisk/test-$CI_JOB_ID + - mkdir -p $TMPDIR - go test -coverprofile=coverage-dax-integration.out -covermode=atomic -coverpkg=${PKG_LIST} -timeout=20m ./dax/test/dax + after_script: + - rm -rf /mnt/ramdisk/test-$CI_JOB_ID artifacts: paths: - coverage-dax-integration.out diff --git a/ctl/dax.go b/ctl/dax.go index 9bee92028..1e2f537e0 100644 --- a/ctl/dax.go +++ b/ctl/dax.go @@ -18,8 +18,21 @@ func BuildDAXFlags(cmd *cobra.Command, srv *server.Command) { flags.BoolVar(&srv.Config.Controller.Run, "controller.run", srv.Config.Controller.Run, "Run the Controller service in process.") flags.DurationVar(&srv.Config.Controller.Config.RegistrationBatchTimeout, "controller.config.registration-batch-timeout", srv.Config.Controller.Config.RegistrationBatchTimeout, "Timeout for node registration batches.") flags.StringVar(&srv.Config.Controller.Config.DataDir, "controller.config.data-dir", srv.Config.Controller.Config.DataDir, "Controller directory to use in process.") + flags.StringVar(&srv.Config.Controller.Config.StorageMethod, "controller.config.storage-method", srv.Config.Controller.Config.StorageMethod, "Backing store. boltdb or sqldb.") flags.DurationVar(&srv.Config.Controller.Config.SnappingTurtleTimeout, "controller.config.snapping-turtle-timeout", srv.Config.Controller.Config.SnappingTurtleTimeout, "Period for running automatic snapshotting routine.") + // Controller.SQLDB + flags.StringVar(&srv.Config.Controller.Config.SQLDB.Database, "controller.config.sqldb.database", srv.Config.Controller.Config.SQLDB.Database, "Database name.") + flags.StringVar(&srv.Config.Controller.Config.SQLDB.Host, "controller.config.sqldb.host", srv.Config.Controller.Config.SQLDB.Host, "Hostname of SQL Database") + flags.StringVar(&srv.Config.Controller.Config.SQLDB.Port, "controller.config.sqldb.port", srv.Config.Controller.Config.SQLDB.Port, "Port of SQL Database") + flags.StringVar(&srv.Config.Controller.Config.SQLDB.User, "controller.config.sqldb.user", srv.Config.Controller.Config.SQLDB.User, "Username for connection to SQL Database") + flags.StringVar(&srv.Config.Controller.Config.SQLDB.Password, "controller.config.sqldb.password", srv.Config.Controller.Config.SQLDB.Password, "Password for connection to SQL Database") + flags.StringVar(&srv.Config.Controller.Config.SQLDB.URL, "controller.config.sqldb.url", srv.Config.Controller.Config.SQLDB.URL, "URL for connection to SQL Database (supersedes host/port/username etc)") + flags.IntVar(&srv.Config.Controller.Config.SQLDB.Pool, "controller.config.sqldb.pool", srv.Config.Controller.Config.SQLDB.Pool, "Max number of open connections to database. 0=unlimited") + flags.IntVar(&srv.Config.Controller.Config.SQLDB.IdlePool, "controller.config.sqldb.idle-pool", srv.Config.Controller.Config.SQLDB.IdlePool, "Maximum number of idle connections to database.") + flags.DurationVar(&srv.Config.Controller.Config.SQLDB.ConnMaxLifetime, "controller.config.sqldb.conn-max-lifetime", srv.Config.Controller.Config.SQLDB.ConnMaxLifetime, "See https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime") + flags.DurationVar(&srv.Config.Controller.Config.SQLDB.ConnMaxIdleTime, "controller.config.sqldb.conn-max-idle-time", srv.Config.Controller.Config.SQLDB.ConnMaxIdleTime, "See https://golang.org/pkg/database/sql/#DB.SetConnMaxIdletime") + // Queryer flags.BoolVar(&srv.Config.Queryer.Run, "queryer.run", srv.Config.Queryer.Run, "Run the Queryer service in process.") flags.StringVar(&srv.Config.Queryer.Config.ControllerAddress, "queryer.config.controller-address", srv.Config.Queryer.Config.ControllerAddress, "Address of remote Controller process.") diff --git a/dax/Makefile b/dax/Makefile index 158e7d8d1..ae1fde873 100644 --- a/dax/Makefile +++ b/dax/Makefile @@ -20,6 +20,11 @@ test-integration: testv-integration: $(GO) test -v ./test/dax -count 1 -timeout 20m -run TestDAXIntegration/$(RUN) +######################## Postgres Backend Stuff ######################### + +run-postgres: + mkdir -p $(HOME)/pgdata + docker run -p 5432:5432 -v $(HOME)/pgdata:/var/lib/postgresql/data:Z -e POSTGRES_PASSWORD=testpass --user=$(shell id -u) postgres:14.7 -c log_min_duration_statement=0 ############################### AWS STUFF ############################### diff --git a/dax/boltdb/boltdb.go b/dax/boltdb/boltdb.go index aa50bd27d..78717ada1 100644 --- a/dax/boltdb/boltdb.go +++ b/dax/boltdb/boltdb.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/featurebasedb/featurebase/v3/dax" "github.com/featurebasedb/featurebase/v3/errors" bolt "go.etcd.io/bbolt" ) @@ -126,7 +127,7 @@ func (db *DB) Close() (err error) { // provides a reference to the database and a fixed timestamp at the start of // the transaction. The timestamp allows us to mock time during tests as well. // The wrapper also contains the context. -func (db *DB) BeginTx(ctx context.Context, writable bool) (*Tx, error) { +func (db *DB) BeginTx(ctx context.Context, writable bool) (dax.Transaction, error) { tx, err := db.db.Begin(writable) if err != nil { return nil, err diff --git a/dax/boltdb/directiveversion.go b/dax/boltdb/directiveversion.go index c4c04fe44..b4a900f07 100644 --- a/dax/boltdb/directiveversion.go +++ b/dax/boltdb/directiveversion.go @@ -34,7 +34,7 @@ func NewDirectiveVersion(db *DB) *DirectiveVersion { func (d *DirectiveVersion) Increment(tx dax.Transaction, delta uint64) (uint64, error) { txx, ok := tx.(*Tx) if !ok { - return 0, dax.NewErrInvalidTransaction() + return 0, dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketDirective) diff --git a/dax/controller/balancer.go b/dax/controller/balancer.go index 3884bd997..87edb3cfd 100644 --- a/dax/controller/balancer.go +++ b/dax/controller/balancer.go @@ -13,6 +13,9 @@ type Balancer interface { // be either transferred to other workers or placed on the free job list. RemoveWorker(tx dax.Transaction, addr dax.Address) ([]dax.WorkerDiff, error) + // FreeWorkers dissociates the given workers from a database. + FreeWorkers(tx dax.Transaction, addrs ...dax.Address) error + // AddJobs adds new jobs for the given database. AddJobs(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID, jobs ...dax.Job) ([]dax.WorkerDiff, error) @@ -61,6 +64,9 @@ func (b *NopBalancer) AddWorker(tx dax.Transaction, node *dax.Node) ([]dax.Worke func (b *NopBalancer) RemoveWorker(tx dax.Transaction, addr dax.Address) ([]dax.WorkerDiff, error) { return []dax.WorkerDiff{}, nil } +func (b *NopBalancer) FreeWorkers(tx dax.Transaction, addrs ...dax.Address) error { + return nil +} func (b *NopBalancer) AddJobs(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID, jobs ...dax.Job) ([]dax.WorkerDiff, error) { return []dax.WorkerDiff{}, nil } diff --git a/dax/controller/balancer/balancer.go b/dax/controller/balancer/balancer.go index 71609b39b..b638b8d22 100644 --- a/dax/controller/balancer/balancer.go +++ b/dax/controller/balancer/balancer.go @@ -108,6 +108,7 @@ func (b *Balancer) addWorker(tx dax.Transaction, roleType dax.RoleType, addr dax } func (b *Balancer) assignMinWorkers(tx dax.Transaction, roleType dax.RoleType) (InternalDiffs, error) { + b.logger.Debugf("assigning min workers for '%s'", roleType) // Find out how many free workers we have. freeWorkers, err := b.freeWorkers.ListWorkers(tx, roleType) if err != nil { @@ -117,6 +118,7 @@ func (b *Balancer) assignMinWorkers(tx dax.Transaction, roleType dax.RoleType) ( // If there are no free workers, return early. if freeWorkerCount == 0 { + b.logger.Debugf("No free workers for '%s'", roleType) return InternalDiffs{}, nil } @@ -176,7 +178,7 @@ func (b *Balancer) assignMinWorkers(tx dax.Transaction, roleType dax.RoleType) ( // For each database, if there are enough free workers to // satisfy its min, then pop that number of workers from the free list. If - // not, contine to the next database until either reaching the end of the + // not, continue to the next database until either reaching the end of the // database list or until there are no more free workers in the list, // whichever comes first. for _, qdbid := range qdbids { @@ -242,6 +244,21 @@ func (b *Balancer) databaseHasJobs(tx dax.Transaction, roleType dax.RoleType, qd func (b *Balancer) RemoveWorker(tx dax.Transaction, addr dax.Address) ([]dax.WorkerDiff, error) { diffs := NewInternalDiffs() + ////// The rest is database specific. //////////// + + // See if the worker is assigned to a database. If it's not, return early. + dbkey := b.current.DatabaseForWorker(tx, addr) + if dbkey != "" { + qdbid := dbkey.QualifiedDatabaseID() + for _, rt := range []dax.RoleType{dax.RoleTypeCompute, dax.RoleTypeTranslate} { + if diff, err := b.removeDatabaseWorker(tx, rt, qdbid, addr); err != nil { + return nil, errors.Wrapf(err, "removing worker: (%s) %s", rt, addr) + } else { + diffs.Merge(diff) + } + } + } + // Remove the worker from the free worker list (if it's there). for _, rt := range []dax.RoleType{dax.RoleTypeCompute, dax.RoleTypeTranslate} { if err := b.freeWorkers.RemoveWorker(tx, rt, addr); err != nil { @@ -254,31 +271,16 @@ func (b *Balancer) RemoveWorker(tx dax.Transaction, addr dax.Address) ([]dax.Wor return nil, errors.Wrapf(err, "deleting node from node service: %s", addr) } - ////// The rest is database specific. //////////// - - // See if the worker is assigned to a database. If it's not, return early. - dbkey := b.current.DatabaseForWorker(tx, addr) - if dbkey == "" { - return diffs.Output(), nil - } - - qdbid := dbkey.QualifiedDatabaseID() - - for _, rt := range []dax.RoleType{dax.RoleTypeCompute, dax.RoleTypeTranslate} { - if diff, err := b.removeDatabaseWorker(tx, rt, qdbid, addr); err != nil { - return nil, errors.Wrapf(err, "removing worker: (%s) %s", rt, addr) + if dbkey != "" { + qdbid := dbkey.QualifiedDatabaseID() + // Balance the affected database. + if diff, err := b.balanceDatabase(tx, qdbid); err != nil { + return nil, errors.Wrapf(err, "balancing database: %s", qdbid) } else { diffs.Merge(diff) } } - // Balance the affected database. - if diff, err := b.balanceDatabase(tx, qdbid); err != nil { - return nil, errors.Wrapf(err, "balancing database: %s", qdbid) - } else { - diffs.Merge(diff) - } - return diffs.Output(), nil } @@ -309,6 +311,10 @@ func (b *Balancer) removeDatabaseWorker(tx dax.Transaction, roleType dax.RoleTyp return diff, nil } +func (b *Balancer) FreeWorkers(tx dax.Transaction, addrs ...dax.Address) error { + return errors.Wrap(b.current.FreeWorkers(tx, addrs...), "freeing workers") +} + func (b *Balancer) AddJobs(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID, jobs ...dax.Job) ([]dax.WorkerDiff, error) { start := time.Now() defer func() { @@ -533,6 +539,7 @@ func (b *Balancer) balanceDatabase(tx dax.Transaction, qdbid dax.QualifiedDataba } func (b *Balancer) balanceDatabaseForRole(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (InternalDiffs, error) { + b.logger.Debugf("balancing database %s for role: %s\n", qdbid, roleType) diffs := NewInternalDiffs() // Before balancing, make sure the database has its minimum number of @@ -776,16 +783,13 @@ func (b *Balancer) processFreeJobs(tx dax.Transaction, roleType dax.RoleType, qd if err != nil { return nil, errors.Wrapf(err, "listing free jobs: %s", roleType) } - for _, job := range jobs { - if aj, err := b.addDatabaseJobs(tx, roleType, qdbid, job); err != nil { - return nil, errors.Wrapf(err, "adding job: %s", job) - } else { - diffs.Merge(aj) - } - if err := b.freeJobs.DeleteJob(tx, roleType, qdbid, job); err != nil { - return nil, errors.Wrapf(err, "deleting free job: %s", job) - } + + if aj, err := b.addDatabaseJobs(tx, roleType, qdbid, jobs...); err != nil { + return nil, errors.Wrapf(err, "adding jobs: %s", jobs) + } else { + diffs.Merge(aj) } + return diffs, nil } @@ -820,6 +824,7 @@ type WorkerJobService interface { CreateWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error DeleteWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error + FreeWorkers(tx dax.Transaction, addrs ...dax.Address) error CreateJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, job ...dax.Job) error DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, job dax.Job) error diff --git a/dax/controller/balancer/balancer_test.go b/dax/controller/balancer/balancer_test.go index 51078c901..4db187e8b 100644 --- a/dax/controller/balancer/balancer_test.go +++ b/dax/controller/balancer/balancer_test.go @@ -15,12 +15,13 @@ import ( testbolt "github.com/featurebasedb/featurebase/v3/dax/test/boltdb" "github.com/featurebasedb/featurebase/v3/logger" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func newBoltBalancer(t *testing.T) (*daxbolt.DB, func()) { db := testbolt.MustOpenDB(t) - assert.NoError(t, db.InitializeBuckets(boltdb.BalancerBuckets...)) - assert.NoError(t, db.InitializeBuckets(schemardb.SchemarBuckets...)) + require.NoError(t, db.InitializeBuckets(boltdb.BalancerBuckets...)) + require.NoError(t, db.InitializeBuckets(schemardb.SchemarBuckets...)) return db, func() { testbolt.MustCloseDB(t, db) diff --git a/dax/controller/balancer/boltdb/balancer.go b/dax/controller/balancer/boltdb/balancer.go index e151aa4db..34f89d7b0 100644 --- a/dax/controller/balancer/boltdb/balancer.go +++ b/dax/controller/balancer/boltdb/balancer.go @@ -75,7 +75,7 @@ func (w *workerJobService) ListWorkers(tx dax.Transaction, roleType dax.RoleType func (w *workerJobService) getWorkers(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Addresses, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } c := txx.Bucket(bucketBalancer).Cursor() @@ -104,7 +104,7 @@ func (w *workerJobService) getWorkers(tx dax.Transaction, roleType dax.RoleType, func (w *workerJobService) getWorkerInfos(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.WorkerInfos, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } c := txx.Bucket(bucketBalancer).Cursor() @@ -147,7 +147,7 @@ func (w *workerJobService) getWorkerInfos(tx dax.Transaction, roleType dax.RoleT func (w *workerJobService) CreateWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -176,7 +176,7 @@ func (w *workerJobService) CreateWorker(tx dax.Transaction, roleType dax.RoleTyp func (w *workerJobService) DeleteWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -195,10 +195,14 @@ func (w *workerJobService) DeleteWorker(tx dax.Transaction, roleType dax.RoleTyp return nil } +func (w *workerJobService) FreeWorkers(tx dax.Transaction, addrs ...dax.Address) error { + return nil +} + func (w *workerJobService) CreateJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, jobs ...dax.Job) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -236,7 +240,7 @@ func (w *workerJobService) CreateJobs(tx dax.Transaction, roleType dax.RoleType, func (w *workerJobService) DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, job dax.Job) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -274,7 +278,7 @@ func (w *workerJobService) DeleteJob(tx dax.Transaction, roleType dax.RoleType, func (w *workerJobService) DeleteJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) (balancer.InternalDiffs, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -323,7 +327,7 @@ func (w *workerJobService) DeleteJobsForTable(tx dax.Transaction, roleType dax.R func (w *workerJobService) ListJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) (dax.Jobs, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -349,7 +353,7 @@ func (w *workerJobService) ListJobs(tx dax.Transaction, roleType dax.RoleType, q func (w *workerJobService) JobCounts(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addrs ...dax.Address) (map[dax.Address]int, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -466,7 +470,7 @@ func (f *freeJobService) CreateJobs(tx dax.Transaction, roleType dax.RoleType, q func (f *freeJobService) DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, job dax.Job) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -504,7 +508,7 @@ func (f *freeJobService) DeleteJob(tx dax.Transaction, roleType dax.RoleType, qd func (f *freeJobService) DeleteJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -542,7 +546,7 @@ func (f *freeJobService) DeleteJobsForTable(tx dax.Transaction, roleType dax.Rol func (f *freeJobService) ListJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Jobs, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -568,7 +572,7 @@ func (f *freeJobService) ListJobs(tx dax.Transaction, roleType dax.RoleType, qdb func (f *freeJobService) MergeJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs dax.Jobs) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -621,7 +625,7 @@ func newFreeWorkerService(db *boltdb.DB) *freeWorkerService { func (f *freeWorkerService) AddWorkers(tx dax.Transaction, roleType dax.RoleType, addres ...dax.Address) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -659,7 +663,7 @@ func (f *freeWorkerService) AddWorkers(tx dax.Transaction, roleType dax.RoleType func (f *freeWorkerService) RemoveWorker(tx dax.Transaction, roleType dax.RoleType, addr dax.Address) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } workers, err := f.ListWorkers(tx, roleType) @@ -701,7 +705,7 @@ func (f *freeWorkerService) RemoveWorker(tx dax.Transaction, roleType dax.RoleTy func (f *freeWorkerService) PopWorkers(tx dax.Transaction, roleType dax.RoleType, num int) ([]dax.Address, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } workers, err := f.ListWorkers(tx, roleType) @@ -743,7 +747,7 @@ func (f *freeWorkerService) PopWorkers(tx dax.Transaction, roleType dax.RoleType func (f *freeWorkerService) ListWorkers(tx dax.Transaction, roleType dax.RoleType) (dax.Addresses, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) diff --git a/dax/controller/balancer/boltdb/node.go b/dax/controller/balancer/boltdb/node.go index f497982f0..9a3624068 100644 --- a/dax/controller/balancer/boltdb/node.go +++ b/dax/controller/balancer/boltdb/node.go @@ -33,7 +33,7 @@ func NewNodeService(db *boltdb.DB, logger logger.Logger) *NodeService { func (s *NodeService) CreateNode(tx dax.Transaction, addr dax.Address, node *dax.Node) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -56,7 +56,7 @@ func (s *NodeService) CreateNode(tx dax.Transaction, addr dax.Address, node *dax func (s *NodeService) ReadNode(tx dax.Transaction, addr dax.Address) (*dax.Node, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -80,7 +80,7 @@ func (s *NodeService) ReadNode(tx dax.Transaction, addr dax.Address) (*dax.Node, func (s *NodeService) DeleteNode(tx dax.Transaction, addr dax.Address) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } bkt := txx.Bucket(bucketBalancer) @@ -98,7 +98,7 @@ func (s *NodeService) DeleteNode(tx dax.Transaction, addr dax.Address) error { func (s *NodeService) Nodes(tx dax.Transaction) ([]*dax.Node, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } nodes, err := s.getNodes(txx) diff --git a/dax/controller/balancer/free_job_test.go b/dax/controller/balancer/free_job_test.go new file mode 100644 index 000000000..dd72fb848 --- /dev/null +++ b/dax/controller/balancer/free_job_test.go @@ -0,0 +1,81 @@ +package balancer_test + +import ( + "context" + "testing" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/controller/sqldb" + "github.com/stretchr/testify/require" +) + +func TestFreeJobService(t *testing.T) { + tx, err := SQLTransactor.BeginTx(context.Background(), true) + require.NoError(t, err, "getting transaction") + + defer func() { + err := tx.Rollback() + if err != nil { + t.Logf("rolling back: %v", err) + } + }() + + // must have a database to do job stuff + schemar := sqldb.NewSchemar(nil) + err = schemar.CreateDatabase(tx, + &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: dax.Database{ID: dbID, Name: dbName}}) + require.NoError(t, err) + + fjSvc := sqldb.NewFreeJobService(nil) + qdbid := dax.QualifiedDatabaseID{OrganizationID: orgID, DatabaseID: dbID} + qtid := dax.QualifiedTableID{ + QualifiedDatabaseID: qdbid, + Name: tableName, + ID: tableID, + } + job1 := dax.Job(qtid.Key() + "job1") + job2 := dax.Job(qtid.Key() + "job2") + job3 := dax.Job(qtid.Key() + "job3") + + err = fjSvc.CreateJobs(tx, role, qdbid, job1, job2, job3) + require.NoError(t, err) + + err = fjSvc.DeleteJob(tx, role, qdbid, job2) + require.NoError(t, err) + + jobs, err := fjSvc.ListJobs(tx, role, qdbid) + require.NoError(t, err) + require.ElementsMatch(t, dax.Jobs{job1, job3}, jobs) + + fwSvc := sqldb.NewFreeWorkerService(nil) + err = fwSvc.AddWorkers(tx, role, nodeAddr) + require.NoError(t, err) + + wjSvc := sqldb.NewWorkerJobService(nil) + err = wjSvc.CreateWorker(tx, role, qdbid, nodeAddr) + require.NoError(t, err) + + err = wjSvc.CreateJobs(tx, role, qdbid, nodeAddr, job1) + require.NoError(t, err) + + jobs, err = fjSvc.ListJobs(tx, role, qdbid) + require.NoError(t, err) + require.ElementsMatch(t, dax.Jobs{job3}, jobs) + + err = fjSvc.MergeJobs(tx, role, qdbid, dax.Jobs{job1}) + require.NoError(t, err) + + jobs, err = fjSvc.ListJobs(tx, role, qdbid) + require.NoError(t, err) + require.ElementsMatch(t, dax.Jobs{job1, job3}, jobs) + + err = fjSvc.DeleteJobsForTable(tx, role, qtid) + require.NoError(t, err) + + jobs, err = fjSvc.ListJobs(tx, role, qdbid) + require.NoError(t, err) + require.Empty(t, jobs) + +} diff --git a/dax/controller/balancer/free_worker_test.go b/dax/controller/balancer/free_worker_test.go new file mode 100644 index 000000000..56dff94d6 --- /dev/null +++ b/dax/controller/balancer/free_worker_test.go @@ -0,0 +1,38 @@ +package balancer_test + +import ( + "context" + "testing" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/controller/sqldb" + "github.com/stretchr/testify/require" +) + +func TestFreeWorkerService(t *testing.T) { + tx, err := SQLTransactor.BeginTx(context.Background(), true) + require.NoError(t, err, "getting transaction") + + defer func() { + err := tx.Rollback() + if err != nil { + t.Logf("rolling back: %v", err) + } + }() + + fwSvc := sqldb.NewFreeWorkerService(nil) + err = fwSvc.AddWorkers(tx, role, nodeAddr, nodeAddr2, nodeAddr3, nodeAddr4, nodeAddr5) + require.NoError(t, err) + + err = fwSvc.RemoveWorker(tx, role, nodeAddr2) + require.NoError(t, err) + + addrs, err := fwSvc.ListWorkers(tx, role) + require.NoError(t, err) + require.ElementsMatch(t, dax.Addresses{nodeAddr, nodeAddr3, nodeAddr4, nodeAddr5}, addrs) + + addrs, err = fwSvc.PopWorkers(tx, role, 2) + require.NoError(t, err) + require.Equal(t, 2, len(addrs)) + require.NotEqual(t, addrs[0], addrs[1]) +} diff --git a/dax/controller/balancer/node_test.go b/dax/controller/balancer/node_test.go new file mode 100644 index 000000000..186b31bb3 --- /dev/null +++ b/dax/controller/balancer/node_test.go @@ -0,0 +1,65 @@ +package balancer_test + +import ( + "context" + "testing" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/controller/sqldb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + nodeAddr = "myaddress" + nodeAddr2 = "myaddress2" + nodeAddr3 = "myaddress3" + nodeAddr4 = "myaddress4" + nodeAddr5 = "myaddress5" +) + +func TestNodeService(t *testing.T) { + tx, err := SQLTransactor.BeginTx(context.Background(), true) + require.NoError(t, err, "getting transaction") + + defer func() { + err := tx.Rollback() + if err != nil { + t.Logf("rolling back: %v", err) + } + }() + + nodeSvc := sqldb.NewNodeService(nil) + + err = nodeSvc.CreateNode(tx, dax.Address(""), &dax.Node{Address: nodeAddr, RoleTypes: []dax.RoleType{"compute"}}) + require.NoError(t, err) + + node, err := nodeSvc.ReadNode(tx, nodeAddr) + require.NoError(t, err) + require.EqualValues(t, nodeAddr, node.Address) + require.EqualValues(t, 1, len(node.RoleTypes)) + require.EqualValues(t, "compute", node.RoleTypes[0]) + + err = nodeSvc.CreateNode(tx, dax.Address(""), &dax.Node{Address: nodeAddr2, RoleTypes: []dax.RoleType{"translate", "compute"}}) + require.NoError(t, err, "create node 2") + + err = nodeSvc.CreateNode(tx, dax.Address(""), &dax.Node{Address: nodeAddr3, RoleTypes: []dax.RoleType{"compute"}}) + require.NoError(t, err, "create node 3") + + nodes, err := nodeSvc.Nodes(tx) + require.NoError(t, err) + assert.EqualValues(t, 3, len(nodes)) + for _, node := range nodes { + assert.Contains(t, node.RoleTypes, dax.RoleType("compute"), "node should have compute role but is: %+v", node) + } + + err = nodeSvc.DeleteNode(tx, nodeAddr2) + require.NoError(t, err, "deleting node") + + nodes, err = nodeSvc.Nodes(tx) + require.NoError(t, err) + require.EqualValues(t, 2, len(nodes)) + for _, node := range nodes { + require.Contains(t, node.RoleTypes, dax.RoleType("compute"), "node should have compute role but is: %+v", node) + } +} diff --git a/dax/controller/balancer/sqldb_init_test.go b/dax/controller/balancer/sqldb_init_test.go new file mode 100644 index 000000000..72bb910b8 --- /dev/null +++ b/dax/controller/balancer/sqldb_init_test.go @@ -0,0 +1,36 @@ +package balancer_test + +import ( + "fmt" + "os" + "testing" + + "github.com/featurebasedb/featurebase/v3/dax/controller/sqldb" + "github.com/featurebasedb/featurebase/v3/logger" +) + +func TestMain(m *testing.M) { + os.Exit(run(m)) +} + +// SQLTransactor is a global connection to a SQL database which is +// created, migrated, and destroyed for each test run. The database +// gets a randomized name and is used by all the tests in this +// package. +var SQLTransactor sqldb.Transactor + +// run is a separate function so that we can defer cleanups. (deferred functions won't run if os.Exit is called) +func run(m *testing.M) int { + // We connect to a randomized database, create it, and run migrations. Then we drop it when tests are done. + conf := sqldb.GetTestConfigRandomDB("balancer_test") + var err error + SQLTransactor, err = sqldb.Connect(conf, logger.StderrLogger) + if err != nil { + fmt.Printf("couldn't make connection to database: %v", err) + return -1 + } + + defer sqldb.DropDatabase(SQLTransactor) + code := m.Run() + return code +} diff --git a/dax/controller/balancer/worker_job_test.go b/dax/controller/balancer/worker_job_test.go new file mode 100644 index 000000000..f5ada4ab4 --- /dev/null +++ b/dax/controller/balancer/worker_job_test.go @@ -0,0 +1,119 @@ +package balancer_test + +import ( + "context" + "testing" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/controller/sqldb" + "github.com/stretchr/testify/require" +) + +const ( + orgID = "orgid" + dbID = "blah" + dbName = "nameofdb" + role = "compute" + tableName = "tbl" + tableID = "tblid" +) + +func TestWorkerJobService(t *testing.T) { + tx, err := SQLTransactor.BeginTx(context.Background(), true) + require.NoError(t, err, "getting transaction") + + defer func() { + err := tx.Rollback() + if err != nil { + t.Logf("rolling back: %v", err) + } + }() + + // must have a database to do workerjob stuff + schemar := sqldb.NewSchemar(nil) + err = schemar.CreateDatabase(tx, + &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: dax.Database{ID: dbID, Name: dbName}}) + require.NoError(t, err) + + wjSvc := sqldb.NewWorkerJobService(nil) + qdbid := dax.QualifiedDatabaseID{OrganizationID: orgID, DatabaseID: dbID} + + // have to create a free worker before you can create a worker job worker + fwSvc := sqldb.NewFreeWorkerService(nil) + err = fwSvc.AddWorkers(tx, role, nodeAddr) + require.NoError(t, err) + + err = wjSvc.CreateWorker(tx, role, qdbid, nodeAddr) + require.NoError(t, err) + + // we create a qtid to prefix jobs so that we can then test the + // "DeleteJobsForTable" method. Is it strange that a table is not + // explicitly mentioned in the interface on the way in, but is in + // the Delete method? Why yes, yes it is... thank you for asking. #TODO + qtid := dax.QualifiedTableID{ + QualifiedDatabaseID: qdbid, + Name: tableName, + ID: tableID, + } + job1 := dax.Job(qtid.Key() + "job1") + job2 := dax.Job(qtid.Key() + "job2") + job3 := dax.Job(qtid.Key() + "job3") + + fjSvc := sqldb.NewFreeJobService(nil) + err = fjSvc.CreateJobs(tx, role, qdbid, job1, job2, job3) + require.NoError(t, err) + + err = wjSvc.CreateJobs(tx, role, qdbid, nodeAddr, job1, job2) + require.NoError(t, err) + + jobs, err := wjSvc.ListJobs(tx, role, qdbid, nodeAddr) + require.NoError(t, err) + require.ElementsMatch(t, dax.Jobs{job1, job2}, jobs) + + workerInfos, err := wjSvc.WorkersJobs(tx, role, qdbid) + require.NoError(t, err) + require.Equal(t, 1, len(workerInfos)) + require.ElementsMatch(t, []dax.Job{job1, job2}, workerInfos[0].Jobs) + + cnt, err := wjSvc.WorkerCount(tx, role, qdbid) + require.NoError(t, err) + require.Equal(t, 1, cnt) + + addrs, err := wjSvc.ListWorkers(tx, role, qdbid) + require.NoError(t, err) + require.ElementsMatch(t, dax.Addresses{nodeAddr}, addrs) + + err = wjSvc.CreateJobs(tx, role, qdbid, nodeAddr, job3) + require.NoError(t, err) + + jcs, err := wjSvc.JobCounts(tx, role, qdbid, nodeAddr) + require.NoError(t, err) + require.Equal(t, 3, jcs[nodeAddr]) + + err = wjSvc.DeleteJob(tx, role, qdbid, nodeAddr, job3) + require.NoError(t, err) + + idiffs, err := wjSvc.DeleteJobsForTable(tx, role, qtid) + require.NoError(t, err) + workerDiffs := idiffs.Output() + require.Equal(t, 1, len(workerDiffs)) + require.EqualValues(t, nodeAddr, workerDiffs[0].Address) + require.Empty(t, workerDiffs[0].AddedJobs) + require.ElementsMatch(t, []dax.Job{job1, job2}, workerDiffs[0].RemovedJobs) + + jobs, err = wjSvc.ListJobs(tx, role, qdbid, nodeAddr) + require.NoError(t, err) + require.Empty(t, jobs) + + dk := wjSvc.DatabaseForWorker(tx, nodeAddr) + require.EqualValues(t, "db__orgid__blah", dk) + + err = wjSvc.DeleteWorker(tx, role, qdbid, nodeAddr) + require.NoError(t, err) + + addrs, err = wjSvc.ListWorkers(tx, role, qdbid) + require.NoError(t, err) + require.Empty(t, addrs) +} diff --git a/dax/controller/config.go b/dax/controller/config.go index a39c5979e..ef8bfef5a 100644 --- a/dax/controller/config.go +++ b/dax/controller/config.go @@ -8,6 +8,10 @@ import ( type NewBalancerFn func(string, logger.Logger) Balancer +// TODO honestly, I think a lot of this stuff should be moved into the +// service package. It's config for specific implementations of things +// that are going to be injected into the controller. + type Config struct { Director Director @@ -15,9 +19,11 @@ type Config struct { PollInterval time.Duration `toml:"poll-interval"` // Storage - StorageMethod string `toml:"-"` + StorageMethod string `toml:"storage-method"` DataDir string `toml:"-"` + SQLDB *SQLDBConfig `toml:"sqldb"` + SnapshotterDir string `toml:"snapshotter-dir"` WriteloggerDir string `toml:"writelogger-dir"` @@ -36,3 +42,41 @@ type Config struct { Logger logger.Logger `toml:"-"` } + +type SQLDBConfig struct { + // Dialect is the pop dialect to use. Example: "postgres" or "sqlite3" or "mysql" + Dialect string + // The name of your database. Example: "foo_development" + Database string + // The host of your database. Example: "127.0.0.1" + Host string + // The port of your database. Example: 1234 + // Will default to the "default" port for each dialect. + Port string + // The username of the database user. Example: "root" + User string + // The password of the database user. Example: "password" + Password string + // Instead of specifying each individual piece of the + // connection you can instead just specify the URL of the + // database. Example: "postgres://postgres:postgres@localhost:5432/pop_test?sslmode=disable" + URL string + // Defaults to 0 "unlimited". See https://golang.org/pkg/database/sql/#DB.SetMaxOpenConns + Pool int + // Defaults to 2. See https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns + IdlePool int + // Defaults to 0 "unlimited". See https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime + ConnMaxLifetime time.Duration + // Defaults to 0 "unlimited". See https://golang.org/pkg/database/sql/#DB.SetConnMaxIdleTime + ConnMaxIdleTime time.Duration +} + +func NewSQLDBConfig() *SQLDBConfig { + return &SQLDBConfig{ + Dialect: "postgres", + Database: "controller", + Host: "127.0.0.1", + Pool: 0, + IdlePool: 2, + } +} diff --git a/dax/controller/controller.go b/dax/controller/controller.go index 9d1539a84..2c1323b65 100644 --- a/dax/controller/controller.go +++ b/dax/controller/controller.go @@ -8,7 +8,6 @@ import ( "time" "github.com/featurebasedb/featurebase/v3/dax" - "github.com/featurebasedb/featurebase/v3/dax/boltdb" "github.com/featurebasedb/featurebase/v3/dax/computer" "github.com/featurebasedb/featurebase/v3/dax/controller/poller" "github.com/featurebasedb/featurebase/v3/dax/controller/schemar" @@ -30,11 +29,7 @@ type Controller struct { Schemar schemar.Schemar Balancer Balancer - // Because we stopped using a storage method interface, and always use bolt, - // we need to be sure to close the boltDBs that are created in controller.New() - // whenever controller.Stop() is called. These are pointers to that DB so we can - // close it. - BoltDB *boltdb.DB + Transactor Transactor Snapshotter *snapshotter.Snapshotter Writelogger *writelogger.Writelogger @@ -52,6 +47,8 @@ type Controller struct { snapControl chan struct{} stopping chan struct{} + backgroundGroup errgroup.Group + logger logger.Logger } @@ -105,10 +102,16 @@ func New(cfg Config) *Controller { // Start starts long running subroutines. func (c *Controller) Start() error { - c.poller.Run() - go c.nodeRegistrationRoutine(c.nodeChan, c.registrationBatchTimeout) - go c.snappingTurtleRoutine(c.snappingTurtleTimeout, c.snapControl, c.logger.WithPrefix("Snapping Turtle: ")) + c.backgroundGroup.Go(c.poller.Run) // TODO: this could just use c.stopping as well? + + c.backgroundGroup.Go(func() error { + return c.nodeRegistrationRoutine(c.nodeChan, c.registrationBatchTimeout) + + }) + c.backgroundGroup.Go(func() error { + return c.snappingTurtleRoutine(c.snappingTurtleTimeout, c.snapControl, c.logger.WithPrefix("Snapping Turtle: ")) + }) return nil } @@ -119,7 +122,13 @@ func (c *Controller) Stop() error { close(c.stopping) - return nil + err := c.backgroundGroup.Wait() + err2 := c.Transactor.Close() + if err != nil { + return errors.Wrap(err, "waiting on background routines") + } + + return errors.Wrap(err2, "closing transactor") } // RegisterNodes adds nodes to the controller's list of registered @@ -127,7 +136,7 @@ func (c *Controller) Stop() error { func (c *Controller) RegisterNodes(ctx context.Context, nodes ...*dax.Node) error { c.logger.Printf("c.RegisterNodes(): %s", dax.Nodes(nodes)) - tx, err := c.BoltDB.BeginTx(ctx, true) + tx, err := c.Transactor.BeginTx(ctx, true) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -245,7 +254,7 @@ func (c *Controller) RegisterNode(ctx context.Context, n *dax.Node) error { } } - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -271,7 +280,7 @@ func (c *Controller) CheckInNode(ctx context.Context, n *dax.Node) error { return NewErrNodeKeyInvalid("") } - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -294,7 +303,7 @@ func (c *Controller) CheckInNode(ctx context.Context, n *dax.Node) error { // DeregisterNodes removes nodes from the controller's list of registered nodes. // It sends directives to the removed nodes, but ignores errors. func (c *Controller) DeregisterNodes(ctx context.Context, addresses ...dax.Address) error { - tx, err := c.BoltDB.BeginTx(ctx, true) + tx, err := c.Transactor.BeginTx(ctx, true) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -600,7 +609,7 @@ func (c *Controller) translateWorkersToAssignedNodes(tx dax.Transaction, workers // CreateDatabase adds a database to the schemar. func (c *Controller) CreateDatabase(ctx context.Context, qdb *dax.QualifiedDatabase) error { - tx, err := c.BoltDB.BeginTx(ctx, true) + tx, err := c.Transactor.BeginTx(ctx, true) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -619,7 +628,7 @@ func (c *Controller) CreateDatabase(ctx context.Context, qdb *dax.QualifiedDatab } func (c *Controller) DropDatabase(ctx context.Context, qdbid dax.QualifiedDatabaseID) error { - tx, err := c.BoltDB.BeginTx(ctx, true) + tx, err := c.Transactor.BeginTx(ctx, true) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -644,6 +653,14 @@ func (c *Controller) DropDatabase(ctx context.Context, qdbid dax.QualifiedDataba workerSet.Merge(addrs) } + addrs := make([]dax.Address, 0, len(workerSet)) + for worker := range workerSet { + addrs = append(addrs, worker) + } + if err := c.Balancer.FreeWorkers(tx, addrs...); err != nil { + return errors.Wrap(err, "freeing workers") + } + // Drop the database record from the schema. if err := c.Schemar.DropDatabase(tx, qdbid); err != nil { return errors.Wrap(err, "dropping database from schemar") @@ -670,7 +687,7 @@ func (c *Controller) DropDatabase(ctx context.Context, qdbid dax.QualifiedDataba // DatabaseByName returns the database for the given name. func (c *Controller) DatabaseByName(ctx context.Context, orgID dax.OrganizationID, dbname dax.DatabaseName) (*dax.QualifiedDatabase, error) { - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return nil, errors.Wrap(err, "beginning tx") } @@ -686,7 +703,7 @@ func (c *Controller) DatabaseByName(ctx context.Context, orgID dax.OrganizationI // DatabaseByID returns the database for the given id. func (c *Controller) DatabaseByID(ctx context.Context, qdbid dax.QualifiedDatabaseID) (*dax.QualifiedDatabase, error) { - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return nil, errors.Wrap(err, "beginning tx") } @@ -702,7 +719,7 @@ func (c *Controller) DatabaseByID(ctx context.Context, qdbid dax.QualifiedDataba // SetDatabaseOption sets the option on the given database. func (c *Controller) SetDatabaseOption(ctx context.Context, qdbid dax.QualifiedDatabaseID, option string, value string) error { - tx, err := c.BoltDB.BeginTx(ctx, true) + tx, err := c.Transactor.BeginTx(ctx, true) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -745,7 +762,7 @@ func (c *Controller) Databases(ctx context.Context, orgID dax.OrganizationID, id return nil, dax.NewErrOrganizationIDDoesNotExist(orgID) } - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return nil, errors.Wrap(err, "beginning tx") } @@ -759,7 +776,7 @@ func (c *Controller) Databases(ctx context.Context, orgID dax.OrganizationID, id // affected nodes based on the change. func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) error { c.logger.Debugf("CreateTable %+v", qtbl) - tx, err := c.BoltDB.BeginTx(ctx, true) + tx, err := c.Transactor.BeginTx(ctx, true) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -853,7 +870,7 @@ func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) // DropTable removes a table from the schema and sends directives to all affected // nodes based on the change. func (c *Controller) DropTable(ctx context.Context, qtid dax.QualifiedTableID) error { - tx, err := c.BoltDB.BeginTx(ctx, true) + tx, err := c.Transactor.BeginTx(ctx, true) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -916,7 +933,7 @@ func (c *Controller) dropTable(tx dax.Transaction, qtid dax.QualifiedTableID) (A return nil, errors.Wrapf(err, "dropping table from schemar: %s", qtid) } - // Delete relavent table files from snapshotter and writelogger. + // Delete relevant table files from snapshotter and writelogger. if err := c.Snapshotter.DeleteTable(qtid); err != nil { return nil, errors.Wrap(err, "deleting from snapshotter") } @@ -929,7 +946,7 @@ func (c *Controller) dropTable(tx dax.Transaction, qtid dax.QualifiedTableID) (A // TableByID returns a table by quaified table id. func (c *Controller) TableByID(ctx context.Context, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) { - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return nil, errors.Wrap(err, "beginning tx") } @@ -941,7 +958,7 @@ func (c *Controller) TableByID(ctx context.Context, qtid dax.QualifiedTableID) ( // Tables returns a list of tables by name. func (c *Controller) Tables(ctx context.Context, qdbid dax.QualifiedDatabaseID, ids ...dax.TableID) ([]*dax.QualifiedTable, error) { - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return nil, errors.Wrap(err, "beginning tx") } @@ -954,7 +971,7 @@ func (c *Controller) Tables(ctx context.Context, qdbid dax.QualifiedDatabaseID, // RemoveShards deregisters the table/shard combinations with the controller and // sends the necessary directives. func (c *Controller) RemoveShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) error { - tx, err := c.BoltDB.BeginTx(ctx, true) + tx, err := c.Transactor.BeginTx(ctx, true) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -1244,7 +1261,7 @@ func (c *Controller) SnapshotTable(ctx context.Context, qtid dax.QualifiedTableI // snapshot that shard, then increment its shard version for logs written to the // Writelogger. func (c *Controller) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTableID, shardNum dax.ShardNum) error { - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -1287,7 +1304,7 @@ func (c *Controller) snapshotShardData(tx dax.Transaction, qtid dax.QualifiedTab // partition to snapshot the table keys for that partition, then increment its // version for logs written to the Writelogger. func (c *Controller) SnapshotTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) error { - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -1330,7 +1347,7 @@ func (c *Controller) snapshotTableKeys(tx dax.Transaction, qtid dax.QualifiedTab // to snapshot the keys for that field, then increment its version for logs // written to the Writelogger. func (c *Controller) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName) error { - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -1384,7 +1401,7 @@ func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID } qdbid := qtid.QualifiedDatabaseID - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return nil, errors.Wrap(err, "beginning tx") } @@ -1446,7 +1463,7 @@ func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTable } qdbid := qtid.QualifiedDatabaseID - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return nil, errors.Wrap(err, "beginning tx") } @@ -1521,7 +1538,7 @@ func (c *Controller) IngestPartition(ctx context.Context, qtid dax.QualifiedTabl qdbid := qtid.QualifiedDatabaseID // Try with a read transaction first. - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return "", errors.Wrap(err, "beginning tx") } @@ -1547,7 +1564,7 @@ func (c *Controller) IngestPartition(ctx context.Context, qtid dax.QualifiedTabl if retryAsWrite { tx.Rollback() - tx, err = c.BoltDB.BeginTx(ctx, true) + tx, err = c.Transactor.BeginTx(ctx, true) if err != nil { return "", errors.Wrap(err, "beginning tx") } @@ -1614,7 +1631,7 @@ func (c *Controller) IngestShard(ctx context.Context, qtid dax.QualifiedTableID, qdbid := qtid.QualifiedDatabaseID // Try with a read transaction first. - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return "", errors.Wrap(err, "beginning tx") } @@ -1641,7 +1658,7 @@ func (c *Controller) IngestShard(ctx context.Context, qtid dax.QualifiedTableID, if retryAsWrite { tx.Rollback() - tx, err = c.BoltDB.BeginTx(ctx, true) + tx, err = c.Transactor.BeginTx(ctx, true) if err != nil { return "", errors.Wrap(err, "beginning tx") } @@ -1697,7 +1714,7 @@ func (c *Controller) IngestShard(ctx context.Context, qtid dax.QualifiedTableID, //// func (c *Controller) CreateField(ctx context.Context, qtid dax.QualifiedTableID, fld *dax.Field) error { - tx, err := c.BoltDB.BeginTx(ctx, true) + tx, err := c.Transactor.BeginTx(ctx, true) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -1765,7 +1782,7 @@ func (c *Controller) CreateField(ctx context.Context, qtid dax.QualifiedTableID, } func (c *Controller) DropField(ctx context.Context, qtid dax.QualifiedTableID, fldName dax.FieldName) error { - tx, err := c.BoltDB.BeginTx(ctx, true) + tx, err := c.Transactor.BeginTx(ctx, true) if err != nil { return errors.Wrap(err, "beginning tx") } @@ -1844,7 +1861,7 @@ func (c *Controller) RemoveAddresses(ctx context.Context, addrs ...dax.Address) } func (c *Controller) DebugNodes(ctx context.Context) ([]*dax.Node, error) { - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return nil, errors.Wrap(err, "beginning tx") } @@ -1854,7 +1871,7 @@ func (c *Controller) DebugNodes(ctx context.Context) ([]*dax.Node, error) { } func (c *Controller) CurrentState(ctx context.Context) ([]dax.WorkerInfo, error) { - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return nil, errors.Wrap(err, "beginning tx") } @@ -1878,7 +1895,7 @@ func (c *Controller) sanitizeQTID(tx dax.Transaction, qtid *dax.QualifiedTableID // TableByName gets the full table by name. func (c *Controller) TableByName(ctx context.Context, qdbid dax.QualifiedDatabaseID, name dax.TableName) (*dax.QualifiedTable, error) { - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return nil, errors.Wrap(err, "beginning tx") } @@ -1896,7 +1913,7 @@ func (c *Controller) TableByName(ctx context.Context, qdbid dax.QualifiedDatabas // TableID returns the table id by table name. // TODO(tlt): try to phase this out in favor of TableByName(). func (c *Controller) TableID(ctx context.Context, qdbid dax.QualifiedDatabaseID, name dax.TableName) (dax.QualifiedTableID, error) { - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return dax.QualifiedTableID{}, errors.Wrap(err, "beginning tx") } @@ -1917,7 +1934,7 @@ func (c *Controller) DeleteNode(context.Context, dax.Address) error { return errors.Errorf("Controller.DeleteNode() not implemented") } func (c *Controller) Nodes(ctx context.Context) ([]*dax.Node, error) { - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { return nil, errors.Wrap(err, "beginning tx") } @@ -1925,3 +1942,7 @@ func (c *Controller) Nodes(ctx context.Context) ([]*dax.Node, error) { return c.Balancer.Nodes(tx) } + +func (c *Controller) Logger() logger.Logger { + return c.logger +} diff --git a/dax/controller/controller_test.go b/dax/controller/controller_test.go index e11af1c0e..6dde70bcf 100644 --- a/dax/controller/controller_test.go +++ b/dax/controller/controller_test.go @@ -39,7 +39,7 @@ func TestController(t *testing.T) { cfg := controller.Config{} con := controller.New(cfg) con.Schemar = schemar - con.BoltDB = db + con.Transactor = db con.Director = director // Register a node with an invalid role type. @@ -85,7 +85,7 @@ func TestController(t *testing.T) { con.Balancer = balancerdb.NewBalancer(db, schemar, logger.StderrLogger) con.DirectiveVersion = directivedb.NewDirectiveVersion(db) con.Director = director - con.BoltDB = db + con.Transactor = db var exp []*dax.Directive @@ -599,7 +599,7 @@ func TestController(t *testing.T) { con.Schemar = schemar con.Balancer = balancerdb.NewBalancer(db, schemar, logger.StderrLogger) con.DirectiveVersion = directivedb.NewDirectiveVersion(db) - con.BoltDB = db + con.Transactor = db con.Director = director var exp []*dax.Directive @@ -913,7 +913,7 @@ func TestController(t *testing.T) { con.Schemar = schemar con.Balancer = balancerdb.NewBalancer(db, schemar, logger.StderrLogger) con.DirectiveVersion = directivedb.NewDirectiveVersion(db) - con.BoltDB = db + con.Transactor = db // Register two nodes. node0 := &dax.Node{ diff --git a/dax/controller/node_registerer.go b/dax/controller/node_registerer.go index 4c4ae46ac..ef51f8bcd 100644 --- a/dax/controller/node_registerer.go +++ b/dax/controller/node_registerer.go @@ -19,25 +19,31 @@ import ( // node at a time) multiple directives being sent out serially as each node // joins, and instead tries to handle all new nodes simultaneously. func (c *Controller) nodeRegistrationRoutine(nodes chan *dax.Node, timeout time.Duration) error { + defer c.logger.Printf("Node registration routine complete") if timeout > 0 { + c.logger.Printf("Node registration delayed timeout: %v", timeout) return c.nodeRegistrationDelayed(nodes, timeout) } + c.logger.Printf("Node registration instant") return c.nodeRegistrationInstant(nodes) } func (c *Controller) nodeRegistrationInstant(nodes chan *dax.Node) error { - for node := range nodes { - err := c.RegisterNodes(context.Background(), node) - if err != nil { - c.logger.Errorf("Registering node: %v, encountered error: %v", node, err) + for { + select { + case <-c.stopping: + return nil + case node := <-nodes: + err := c.RegisterNodes(context.Background(), node) + if err != nil { + c.logger.Errorf("Registering node: %v, encountered error: %v", node, err) + } } } - return nil } func (c *Controller) nodeRegistrationDelayed(nodes chan *dax.Node, timeout time.Duration) error { batch := []*dax.Node{} - c.logger.Printf("Running with batch registration timeout: %v", timeout) for { select { case <-c.stopping: @@ -51,7 +57,7 @@ func (c *Controller) nodeRegistrationDelayed(nodes chan *dax.Node, timeout time. if err != nil { c.logger.Errorf("Registering nodes: %v, encountered error: %v", batch, err) } - + c.logger.Printf("Completed registering nodes") batch = batch[:0] // reset batch } } diff --git a/dax/controller/poller/poller.go b/dax/controller/poller/poller.go index 16fcb57e3..c45a83078 100644 --- a/dax/controller/poller/poller.go +++ b/dax/controller/poller/poller.go @@ -21,7 +21,6 @@ type Poller struct { nodePoller NodePoller pollInterval time.Duration - running bool stopping chan struct{} logger logger.Logger @@ -61,7 +60,7 @@ func New(cfg Config) *Poller { func (p *Poller) Addresses() []dax.Address { nodes, err := p.nodeService.Nodes(context.Background()) if err != nil { - p.logger.Printf("POLLER: unable to get nodes from node service") + p.logger.Errorf("POLLER: unable to get nodes from node service: %v", err) } addrs := make([]dax.Address, 0, len(nodes)) @@ -73,17 +72,9 @@ func (p *Poller) Addresses() []dax.Address { } // Run starts the polling goroutine. -func (p *Poller) Run() { - p.mu.Lock() - defer p.mu.Unlock() - - if p.running { - p.logger.Printf("poller is already running") - return - } - p.running = true - - go func() { p.run() }() +func (p *Poller) Run() error { + p.run() + return nil } func (p *Poller) run() { diff --git a/dax/controller/poller/poller_test.go b/dax/controller/poller/poller_test.go index 56746c25f..6c3688a74 100644 --- a/dax/controller/poller/poller_test.go +++ b/dax/controller/poller/poller_test.go @@ -66,7 +66,7 @@ func TestPoller(t *testing.T) { nodeService.CreateNode(ctx, addr1, daxNode1) nodeService.CreateNode(ctx, addr2, daxNode2) - p.Run() + go p.Run() defer p.Stop() // wait for a done diff --git a/dax/controller/schemar/boltdb/schemar.go b/dax/controller/schemar/boltdb/schemar.go index 997a66cf9..7dbe50288 100644 --- a/dax/controller/schemar/boltdb/schemar.go +++ b/dax/controller/schemar/boltdb/schemar.go @@ -68,7 +68,7 @@ func (s *Schemar) CreateDatabase(tx dax.Transaction, qdb *dax.QualifiedDatabase) txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } // Ensure a database with that ID doesn't already exist. @@ -92,7 +92,7 @@ func (s *Schemar) CreateDatabase(tx dax.Transaction, qdb *dax.QualifiedDatabase) func (s *Schemar) DatabaseByID(tx dax.Transaction, qdbid dax.QualifiedDatabaseID) (*dax.QualifiedDatabase, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } return s.databaseByID(txx, qdbid.OrganizationID, qdbid.DatabaseID) @@ -120,7 +120,7 @@ func (s *Schemar) databaseByID(tx *boltdb.Tx, orgID dax.OrganizationID, id dax.D func (s *Schemar) DatabaseByName(tx dax.Transaction, orgID dax.OrganizationID, dbname dax.DatabaseName) (*dax.QualifiedDatabase, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } return s.databaseByName(txx, orgID, dbname) @@ -177,7 +177,7 @@ func (s *Schemar) putDatabaseName(tx *boltdb.Tx, qdb *dax.QualifiedDatabase) err func (s *Schemar) DropDatabase(tx dax.Transaction, qdbid dax.QualifiedDatabaseID) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } // Ensure the database exists. @@ -209,7 +209,7 @@ func (s *Schemar) DropDatabase(tx dax.Transaction, qdbid dax.QualifiedDatabaseID func (s *Schemar) SetDatabaseOption(tx dax.Transaction, qdbid dax.QualifiedDatabaseID, option string, value string) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } // Get the database. @@ -234,7 +234,7 @@ func (s *Schemar) SetDatabaseOption(tx dax.Transaction, qdbid dax.QualifiedDatab func (s *Schemar) Databases(tx dax.Transaction, orgID dax.OrganizationID, ids ...dax.DatabaseID) ([]*dax.QualifiedDatabase, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } return s.getDatabases(txx, orgID, ids...) @@ -326,7 +326,7 @@ func (s *Schemar) CreateTable(tx dax.Transaction, qtbl *dax.QualifiedTable) erro txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } // Ensure the database, defined in the table's QualifiedDatabaseID, exists. @@ -364,7 +364,7 @@ func (s *Schemar) CreateField(tx dax.Transaction, qtid dax.QualifiedTableID, fld txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } // Get the table. @@ -392,7 +392,7 @@ func (s *Schemar) CreateField(tx dax.Transaction, qtid dax.QualifiedTableID, fld func (s *Schemar) DropField(tx dax.Transaction, qtid dax.QualifiedTableID, fldName dax.FieldName) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } // Get the table. @@ -444,7 +444,7 @@ func (s *Schemar) putTableName(tx *boltdb.Tx, qtbl *dax.QualifiedTable) error { func (s *Schemar) Table(tx dax.Transaction, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } return s.tableByQTID(txx, qtid) @@ -507,7 +507,7 @@ func (s *Schemar) tableIDByName(tx *boltdb.Tx, qdbid dax.QualifiedDatabaseID, na func (s *Schemar) Tables(tx dax.Transaction, qdbid dax.QualifiedDatabaseID, ids ...dax.TableID) ([]*dax.QualifiedTable, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return nil, dax.NewErrInvalidTransaction() + return nil, dax.NewErrInvalidTransaction("*boltdb.Tx") } return s.getTables(txx, qdbid, ids...) @@ -572,7 +572,7 @@ func containsTableID(s []dax.TableID, e dax.TableID) bool { func (s *Schemar) DropTable(tx dax.Transaction, qtid dax.QualifiedTableID) error { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.NewErrInvalidTransaction() + return dax.NewErrInvalidTransaction("*boltdb.Tx") } // Ensure the table exists. @@ -602,7 +602,7 @@ func (s *Schemar) DropTable(tx dax.Transaction, qtid dax.QualifiedTableID) error func (s *Schemar) TableID(tx dax.Transaction, qdbid dax.QualifiedDatabaseID, name dax.TableName) (dax.QualifiedTableID, error) { txx, ok := tx.(*boltdb.Tx) if !ok { - return dax.QualifiedTableID{}, dax.NewErrInvalidTransaction() + return dax.QualifiedTableID{}, dax.NewErrInvalidTransaction("*boltdb.Tx") } return s.tableIDByName(txx, qdbid, name) diff --git a/dax/controller/schemar/schemar.go b/dax/controller/schemar/schemar.go index fc57d93c2..3adc589c9 100644 --- a/dax/controller/schemar/schemar.go +++ b/dax/controller/schemar/schemar.go @@ -5,6 +5,8 @@ import ( "github.com/featurebasedb/featurebase/v3/dax" ) +// Schemar is the interface to the schema service which holds all the +// databases and tables stored in the system. type Schemar interface { CreateDatabase(dax.Transaction, *dax.QualifiedDatabase) error DropDatabase(dax.Transaction, dax.QualifiedDatabaseID) error diff --git a/dax/controller/schemar/schemar_test.go b/dax/controller/schemar/schemar_test.go new file mode 100644 index 000000000..684d5955d --- /dev/null +++ b/dax/controller/schemar/schemar_test.go @@ -0,0 +1,339 @@ +package schemar_test + +import ( + "context" + "testing" + + "github.com/featurebasedb/featurebase/v3/dax" + cschemar "github.com/featurebasedb/featurebase/v3/dax/controller/schemar" + "github.com/featurebasedb/featurebase/v3/dax/controller/sqldb" + "github.com/featurebasedb/featurebase/v3/errors" + "github.com/featurebasedb/featurebase/v3/logger" + "github.com/stretchr/testify/require" +) + +const ( + orgID = "orgid" + orgID2 = "orgid2" + dbID = "blah" + dbID2 = "blah2" + dbID3 = "blah3" + dbID4 = "blah4" + dbName = "haha" + dbName2 = "haha2" + tblName = "tbl" +) + +var ( + qdbid = dax.QualifiedDatabaseID{OrganizationID: orgID, DatabaseID: dbID} +) + +// TODO these tests can be generalized to share setup (getting +// transactor/tx), and then run the same tests against multiple +// underlying implementations + +func TestSQLSchemar(t *testing.T) { + conf := sqldb.GetTestConfigRandomDB("sql_schemar") + trans, err := sqldb.Connect(conf, logger.StderrLogger) + require.NoError(t, err, "connecting") + defer sqldb.DropDatabase(trans) + + tx, err := trans.BeginTx(context.Background(), true) + require.NoError(t, err, "getting transaction") + + schemar := sqldb.NewSchemar(nil) + + err = schemar.CreateDatabase(tx, + &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: dax.Database{ID: dbID, Name: dbName}}) + require.NoError(t, err) + + // create 2nd db in same org + err = schemar.CreateDatabase(tx, + &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: dax.Database{ID: dbID2, Name: dbName2}}) + require.NoError(t, err) + + // create 3rd db in new org + schemar.CreateDatabase(tx, + &dax.QualifiedDatabase{ + OrganizationID: orgID2, + Database: dax.Database{ID: dbID3, Name: dbName2}}) + require.NoError(t, err) + + err = schemar.CreateDatabase(tx, + &dax.QualifiedDatabase{OrganizationID: orgID, + Database: dax.Database{ + ID: dbID, + Name: dbName}, + }) + if !errors.Is(err, dax.ErrDatabaseIDExists) { + t.Fatalf("got unexpected error creating DB that already exists: %v", err) + } + + // make sure querying with empty org ID brings back all databases + dbs, err := schemar.Databases(tx, "") + require.NoError(t, err) + require.Equal(t, 3, len(dbs)) + + db, err := schemar.DatabaseByName(tx, orgID, dbName) + require.NoError(t, err) + require.EqualValues(t, dbID, db.ID) + + err = schemar.SetDatabaseOption(tx, qdbid, dax.DatabaseOptionWorkersMax, "4") + require.NoError(t, err) + + err = schemar.SetDatabaseOption(tx, qdbid, dax.DatabaseOptionWorkersMin, "2") + require.NoError(t, err) + + db, err = schemar.DatabaseByID(tx, qdbid) + require.NoError(t, err) + require.EqualValues(t, dbName, db.Name) + require.EqualValues(t, 4, db.Options.WorkersMax) + require.EqualValues(t, 2, db.Options.WorkersMin) + + dbs, err = schemar.Databases(tx, orgID) + require.NoError(t, err) + require.Equal(t, 2, len(dbs)) + require.EqualValues(t, orgID, dbs[0].OrganizationID) + require.EqualValues(t, orgID, dbs[1].OrganizationID) + + dbs, err = schemar.Databases(tx, orgID, dbID) + require.NoError(t, err) + require.Equal(t, 1, len(dbs)) + require.EqualValues(t, orgID, dbs[0].OrganizationID) + require.EqualValues(t, dbID, dbs[0].Database.ID) + + // test create table + qtbl := &dax.QualifiedTable{ + QualifiedDatabaseID: qdbid, + Table: dax.Table{ + Name: tblName, + Fields: []*dax.Field{{ + Name: "_id", + Type: "string", + Options: dax.FieldOptions{}, + }}, + PartitionN: 4, + Description: "desc", + Owner: "own", + UpdatedBy: "me", + }, + } + + _, err = qtbl.CreateID() + require.NoError(t, err) + err = schemar.CreateTable(tx, qtbl) + require.NoError(t, err) + + // test create field + err = schemar.CreateField(tx, qtbl.QualifiedID(), &dax.Field{Name: "age", Type: "int"}) + require.NoError(t, err) + + qtbl, err = schemar.Table(tx, qtbl.QualifiedID()) + require.NoError(t, err) + + require.Equal(t, 2, len(qtbl.Fields)) + + var ageField *dax.Field + + for _, f := range qtbl.Fields { + if f.Name == "age" { + ageField = f + } + } + + require.NotNil(t, ageField) + + // drop field + err = schemar.DropField(tx, qtbl.QualifiedID(), "age") + require.NoError(t, err) + + // ensure field was dropped + qtbl, err = schemar.Table(tx, qtbl.QualifiedID()) + require.NoError(t, err) + require.Equal(t, 1, len(qtbl.Fields)) + + if qtbl.Fields[0].Name != "_id" { + t.Fatalf("unexpected field: %+v", qtbl.Fields[0]) + } + + tables, err := schemar.Tables(tx, qdbid) + require.NoError(t, err) + require.Equal(t, 1, len(tables)) + + // TODO add test for Tables passing table ids + tables, err = schemar.Tables(tx, qdbid, tables[0].ID) + require.NoError(t, err) + require.Equal(t, 1, len(tables)) + + _, err = schemar.TableID(tx, qdbid, tblName) + require.NoError(t, err) + + err = schemar.DropTable(tx, qtbl.QualifiedID()) + require.NoError(t, err) + + // make sure Table was deleted + _, err = schemar.Table(tx, qtbl.QualifiedID()) + require.NotNil(t, err) + + err = schemar.DropDatabase(tx, qdbid) + require.NoError(t, err) + + // make sure DB was deleted + dbs, err = schemar.Databases(tx, orgID) + require.NoError(t, err) + require.Equal(t, 1, len(dbs)) + require.EqualValues(t, dbID2, dbs[0].Database.ID) + + // rollback so we have clean state to test failure cases + err = tx.Rollback() + if err != nil { + require.NoError(t, err, "rolling back to test failure cases") + } + + qtbl = &dax.QualifiedTable{ + QualifiedDatabaseID: qdbid, + Table: dax.Table{ + Name: tblName, + Fields: []*dax.Field{ + { + Name: "_id", + Type: "string", + Options: dax.FieldOptions{}, + }, + { + Name: "age", + Type: "int", + Options: dax.FieldOptions{}, + }, + }, + PartitionN: 4, + Description: "desc", + Owner: "own", + UpdatedBy: "me", + }, + } + qtbl.ID = "" + _, err = qtbl.CreateID() + require.NoError(t, err) + + t.Run("Create Table no DB fails", func(t *testing.T) { + tx2, err := trans.BeginTx(context.Background(), true) + require.NoError(t, err) + defer tx2.Rollback() + err = schemar.CreateTable(tx2, qtbl) + require.NotNil(t, err) + }) + + tx, err = trans.BeginTx(context.Background(), true) + require.NoError(t, err, "beginning transaction") + + err = schemar.CreateDatabase(tx, + &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: dax.Database{ID: dbID, Name: dbName}}) + require.NoError(t, err) + + err = schemar.CreateTable(tx, qtbl) + require.NoError(t, err) + + err = tx.Commit() + require.NoError(t, err) + + t.Run("Drop non-existent field fails with correct error", func(t *testing.T) { + tx, err = trans.BeginTx(context.Background(), true) + require.NoError(t, err) + defer tx.Rollback() + err = schemar.DropField(tx, qtbl.QualifiedID(), "unknownField") + requireCode(t, err, dax.ErrFieldDoesNotExist) + }) + + t.Run("Drop field from non-existent table", func(t *testing.T) { + tx, err = trans.BeginTx(context.Background(), true) + require.NoError(t, err) + defer tx.Rollback() + err = schemar.DropField(tx, dax.QualifiedTableID{QualifiedDatabaseID: qdbid, ID: "blah", Name: "blah"}, "age") + requireCode(t, err, dax.ErrFieldDoesNotExist) + }) + + t.Run("Test Lookup non-existent table fails with correct error (by name)", func(t *testing.T) { + tx, err = trans.BeginTx(context.Background(), true) + require.NoError(t, err) + defer tx.Rollback() + _, err = schemar.Table(tx, dax.QualifiedTableID{QualifiedDatabaseID: qdbid, Name: "humbug"}) + requireCode(t, err, dax.ErrTableNameDoesNotExist) + }) + + t.Run("Test Lookup non-existent table fails with correct error (by name)", func(t *testing.T) { + tx, err = trans.BeginTx(context.Background(), true) + require.NoError(t, err) + defer tx.Rollback() + _, err = schemar.Table(tx, dax.QualifiedTableID{QualifiedDatabaseID: qdbid, ID: "bumhug", Name: "humbug"}) + requireCode(t, err, dax.ErrTableIDDoesNotExist) + }) + + t.Run("Test Lookup non-existent tableID fails with correct error (by name)", func(t *testing.T) { + tx, err = trans.BeginTx(context.Background(), true) + require.NoError(t, err) + defer tx.Rollback() + _, err = schemar.TableID(tx, qdbid, "humbug") + requireCode(t, err, dax.ErrTableNameDoesNotExist) + }) + + t.Run("Test Create existing field fails", func(t *testing.T) { + tx, err = trans.BeginTx(context.Background(), true) + require.NoError(t, err) + defer tx.Rollback() + err = schemar.CreateField(tx, qtbl.QualifiedID(), &dax.Field{Name: "age", Type: "int", Options: dax.FieldOptions{}}) + requireCode(t, err, dax.ErrFieldExists) + }) + + t.Run("Test Create field empty name fails", func(t *testing.T) { + tx, err = trans.BeginTx(context.Background(), true) + require.NoError(t, err) + defer tx.Rollback() + err = schemar.CreateField(tx, qtbl.QualifiedID(), &dax.Field{Name: "", Type: "int", Options: dax.FieldOptions{}}) + requireCode(t, err, cschemar.ErrCodeFieldNameInvalid) + }) + + t.Run("Test create table that already exists", func(t *testing.T) { + tx, err = trans.BeginTx(context.Background(), true) + require.NoError(t, err) + defer tx.Rollback() + err = schemar.CreateTable(tx, qtbl) + requireCode(t, err, dax.ErrTableIDExists) + }) + + t.Run("Find database by name that doesn't exist", func(t *testing.T) { + tx, err = trans.BeginTx(context.Background(), true) + require.NoError(t, err) + defer tx.Rollback() + _, err = schemar.DatabaseByName(tx, orgID, "blooooooo") + requireCode(t, err, dax.ErrDatabaseNameDoesNotExist) + }) + + t.Run("Find database by ID that doesn't exist", func(t *testing.T) { + tx, err = trans.BeginTx(context.Background(), true) + require.NoError(t, err) + defer tx.Rollback() + _, err = schemar.DatabaseByID(tx, dax.QualifiedDatabaseID{OrganizationID: orgID, DatabaseID: "zeeeeeeeeeeeeep"}) + requireCode(t, err, dax.ErrDatabaseIDDoesNotExist) + }) + + t.Run("Drop non-existent database", func(t *testing.T) { + tx, err = trans.BeginTx(context.Background(), true) + require.NoError(t, err) + defer tx.Rollback() + err = schemar.DropDatabase(tx, dax.QualifiedDatabaseID{OrganizationID: orgID, DatabaseID: "yoooo"}) + requireCode(t, err, dax.ErrDatabaseIDDoesNotExist) + }) +} + +func requireCode(t *testing.T, err error, code errors.Code) { + if !errors.Is(err, code) { + t.Fatalf("Error '%v' does not have code %s.", err, code) + } +} diff --git a/dax/controller/service/controller.go b/dax/controller/service/controller.go index 56354888f..0dbc9b0e6 100644 --- a/dax/controller/service/controller.go +++ b/dax/controller/service/controller.go @@ -10,6 +10,7 @@ import ( balancerboltdb "github.com/featurebasedb/featurebase/v3/dax/controller/balancer/boltdb" controllerhttp "github.com/featurebasedb/featurebase/v3/dax/controller/http" schemarboltdb "github.com/featurebasedb/featurebase/v3/dax/controller/schemar/boltdb" + "github.com/featurebasedb/featurebase/v3/dax/controller/sqldb" "github.com/featurebasedb/featurebase/v3/errors" "github.com/featurebasedb/featurebase/v3/logger" fbnet "github.com/featurebasedb/featurebase/v3/net" @@ -22,12 +23,6 @@ type controllerService struct { uri *fbnet.URI controller *controller.Controller - // Because we stopped using a storage method interface, and always use bolt, - // we need to be sure to close the boltDBs that are created in controller.New() - // whenever controller.Stop() is called. These are pointers to that DB so we can - // close it. - boltDB *boltdb.DB - logger logger.Logger } @@ -38,11 +33,6 @@ func New(uri *fbnet.URI, cfg controller.Config) *controllerService { logr = cfg.Logger.WithPrefix("Controller: ") } - // Storage methods. - if cfg.StorageMethod != "boltdb" && cfg.StorageMethod != "" { - logr.Printf("storagemethod %s not supported, try 'boltdb'", cfg.StorageMethod) - } - if cfg.DataDir == "" { dir, err := os.MkdirTemp("", "controller_*") if err != nil { @@ -53,40 +43,53 @@ func New(uri *fbnet.URI, cfg controller.Config) *controllerService { logr.Warnf("no DataDir given (like '/path/to/directory'); using temp dir at '%s'", cfg.DataDir) } - buckets := append(schemarboltdb.SchemarBuckets, balancerboltdb.BalancerBuckets...) + controller := controller.New(cfg) + controllerSvc := &controllerService{ + uri: uri, + controller: controller, + logger: logr, + } - controllerDB, err := boltdb.NewSvcBolt(cfg.DataDir, "controller", buckets...) - if err != nil { - logr.Printf(errors.Wrap(err, "creating controller bolt").Error()) + // Storage methods. + switch cfg.StorageMethod { + case "boltdb": + buckets := append(schemarboltdb.SchemarBuckets, balancerboltdb.BalancerBuckets...) + controllerDB, err := boltdb.NewSvcBolt(cfg.DataDir, "controller", buckets...) + if err != nil { + logr.Printf(errors.Wrap(err, "creating controller bolt").Error()) + os.Exit(1) + } + // Directive version. + if err := controllerDB.InitializeBuckets(boltdb.DirectiveBuckets...); err != nil { + logr.Panicf("initializing directive buckets: %v", err) + } + controller.Schemar = schemarboltdb.NewSchemar(controllerDB, logr) + controller.Balancer = balancerboltdb.NewBalancer(controllerDB, controller.Schemar, logr) + directiveVersion := boltdb.NewDirectiveVersion(controllerDB) + controller.DirectiveVersion = directiveVersion + + controller.Transactor = controllerDB + case "sqldb": + controller.Schemar = sqldb.NewSchemar(logr) + controller.Balancer = sqldb.NewBalancer(logr) + controller.DirectiveVersion = sqldb.NewDirectiveVersion(logr) + + transactor, err := sqldb.Connect(cfg.SQLDB, logr) + if err != nil { + logr.Printf("Connecting to database: %v", err) + os.Exit(1) + } + controller.Transactor = transactor + default: + logr.Printf("storagemethod %s not supported, try 'boltdb' or 'sqldb'", cfg.StorageMethod) os.Exit(1) } - schemar := schemarboltdb.NewSchemar(controllerDB, logr) - balancer := balancerboltdb.NewBalancer(controllerDB, schemar, logr) - - // Directive version. - if err := controllerDB.InitializeBuckets(boltdb.DirectiveBuckets...); err != nil { - logr.Panicf("initializing directive buckets: %v", err) - } - directiveVersion := boltdb.NewDirectiveVersion(controllerDB) - - // Controller. - controller := controller.New(cfg) - controller.Schemar = schemar - controller.Balancer = balancer - controller.DirectiveVersion = directiveVersion - controller.BoltDB = controllerDB - if cfg.Director != nil { controller.Director = cfg.Director } - return &controllerService{ - uri: uri, - controller: controller, - boltDB: controllerDB, - logger: logr, - } + return controllerSvc } func (m *controllerService) Start() error { @@ -103,10 +106,6 @@ func (m *controllerService) Stop() error { m.logger.Warnf("error stopping controller: %v", err) } - if m.boltDB != nil { - m.boltDB.Close() - } - return err } diff --git a/dax/controller/snapping_turtle.go b/dax/controller/snapping_turtle.go index c09a0bbda..03e57852d 100644 --- a/dax/controller/snapping_turtle.go +++ b/dax/controller/snapping_turtle.go @@ -8,9 +8,9 @@ import ( "github.com/featurebasedb/featurebase/v3/logger" ) -func (c *Controller) snappingTurtleRoutine(period time.Duration, control chan struct{}, log logger.Logger) { +func (c *Controller) snappingTurtleRoutine(period time.Duration, control chan struct{}, log logger.Logger) error { if period == 0 { - return // disable automatic snapshotting + return nil } ticker := time.NewTicker(period) for { @@ -18,7 +18,7 @@ func (c *Controller) snappingTurtleRoutine(period time.Duration, control chan st case <-c.stopping: ticker.Stop() log.Debugf("Stopping Snapping Turtle") - return + return nil case <-ticker.C: c.snapAll(log) case <-control: @@ -34,7 +34,7 @@ func (c *Controller) snapAll(log logger.Logger) { }() ctx := context.Background() - tx, err := c.BoltDB.BeginTx(ctx, false) + tx, err := c.Transactor.BeginTx(ctx, false) if err != nil { log.Printf("Error getting transaction for snapping turtle: %v", err) return diff --git a/dax/controller/sqldb/balancer.go b/dax/controller/sqldb/balancer.go new file mode 100644 index 000000000..64833d71e --- /dev/null +++ b/dax/controller/sqldb/balancer.go @@ -0,0 +1,17 @@ +package sqldb + +import ( + "github.com/featurebasedb/featurebase/v3/dax/controller/balancer" + "github.com/featurebasedb/featurebase/v3/logger" +) + +// NewBalancer returns a new instance of controller.Balancer. +func NewBalancer(logger logger.Logger) *balancer.Balancer { + schemar := NewSchemar(logger) + fjs := NewFreeJobService(logger) + wjs := NewWorkerJobService(logger) + fws := NewFreeWorkerService(logger) + ns := NewNodeService(logger) + + return balancer.New(ns, fjs, wjs, fws, schemar, logger) +} diff --git a/dax/controller/sqldb/connect.go b/dax/controller/sqldb/connect.go new file mode 100644 index 000000000..41184f095 --- /dev/null +++ b/dax/controller/sqldb/connect.go @@ -0,0 +1,58 @@ +package sqldb + +import ( + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/controller" + "github.com/featurebasedb/featurebase/v3/errors" + "github.com/featurebasedb/featurebase/v3/logger" + "github.com/gobuffalo/pop/v6" +) + +func Connect(cfg *controller.SQLDBConfig, log logger.Logger) (Transactor, error) { + conn, err := pop.NewConnection(&pop.ConnectionDetails{ + Dialect: cfg.Dialect, + Database: cfg.Database, + Host: cfg.Host, + Port: cfg.Port, + User: cfg.User, + Password: cfg.Password, + URL: cfg.URL, + Pool: cfg.Pool, + IdlePool: cfg.IdlePool, + ConnMaxLifetime: cfg.ConnMaxLifetime, + ConnMaxIdleTime: cfg.ConnMaxIdleTime, + }) + if err != nil { + return Transactor{Connection: nil}, errors.Wrap(err, "creating new connection") + } + + err = pop.CreateDB(conn) + + if err != nil { + log.Warnf("auto-creating database, got error '%v'", err) + } + + err = conn.Open() + if err != nil { + return Transactor{Connection: nil}, errors.Wrap(err, "opening connection") + } + + mig, err := NewEmbedMigrator(dax.MigrationsFS, conn, log) + if err != nil { + return Transactor{Connection: nil}, errors.Wrap(err, "getting embedded migrator") + } + err = mig.Up() + if err != nil { + return Transactor{Connection: nil}, errors.Wrap(err, "migrating DB") + } + + return Transactor{Connection: conn}, nil +} + +// DropDatabase drops the database associated with the given +// Transactor (which embeds a live database connection). This is +// destructive, you will lose data. +func DropDatabase(trans Transactor) error { + conn := trans.Connection + return pop.DropDB(conn) +} diff --git a/dax/controller/sqldb/directiveversion.go b/dax/controller/sqldb/directiveversion.go new file mode 100644 index 000000000..4f776d777 --- /dev/null +++ b/dax/controller/sqldb/directiveversion.go @@ -0,0 +1,35 @@ +package sqldb + +import ( + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/models" + "github.com/featurebasedb/featurebase/v3/errors" + "github.com/featurebasedb/featurebase/v3/logger" +) + +func NewDirectiveVersion(log logger.Logger) dax.DirectiveVersion { + if log == nil { + log = logger.NopLogger + } + return &directiveVersion{ + log: log, + } +} + +type directiveVersion struct { + log logger.Logger +} + +func (d *directiveVersion) Increment(tx dax.Transaction, delta uint64) (uint64, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return 0, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + // table is pre-populated w/ a single record w/ ID=1 during schema migration + dv := &models.DirectiveVersion{} + err := dt.C.RawQuery("UPDATE directive_versions SET version = version + ? WHERE id = ? RETURNING id, version", delta, 1).First(dv) + if err != nil { + return 0, errors.Wrap(err, "updating directive_version") + } + return uint64(dv.Version), nil +} diff --git a/dax/controller/sqldb/freejob.go b/dax/controller/sqldb/freejob.go new file mode 100644 index 000000000..6fbc714cb --- /dev/null +++ b/dax/controller/sqldb/freejob.go @@ -0,0 +1,92 @@ +package sqldb + +import ( + "fmt" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/controller/balancer" + "github.com/featurebasedb/featurebase/v3/dax/models" + "github.com/featurebasedb/featurebase/v3/logger" + "github.com/pkg/errors" +) + +func NewFreeJobService(log logger.Logger) balancer.FreeJobService { + if log == nil { + log = logger.NopLogger + } + return &freeJobService{ + log: log, + } +} + +type freeJobService struct { + log logger.Logger +} + +func (fj *freeJobService) CreateJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, job ...dax.Job) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + jobs := make(models.Jobs, len(job)) + for i, j := range job { + jobs[i] = models.Job{ + Name: j, + Role: roleType, + DatabaseID: qdbid.DatabaseID, + } + } + err := dt.C.Create(jobs) + return errors.Wrap(err, "creating jobs") +} + +func (fj *freeJobService) DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, job dax.Job) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + err := dt.C.RawQuery("DELETE from jobs where role = ? and database_id = ? and name = ? and worker_id is NULL", roleType, qdbid.DatabaseID, job).Exec() + return errors.Wrap(err, "deleting") +} + +func (fj *freeJobService) DeleteJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + err := dt.C.RawQuery("DELETE from jobs where role = ? and database_id = ? and name LIKE ? and worker_id is NULL", + roleType, qtid.DatabaseID, fmt.Sprintf("%s%%", qtid.Key())).Exec() + return errors.Wrap(err, "deleting") +} + +func (fj *freeJobService) ListJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Jobs, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + jobs := make(models.Jobs, 0) + err := dt.C.Where("role = ? and database_id = ? and worker_id is NULL", roleType, qdbid.DatabaseID).Order("name asc").All(&jobs) + if err != nil { + return nil, errors.Wrap(err, "querying for jobs") + } + + djs := make(dax.Jobs, len(jobs)) + for i, job := range jobs { + djs[i] = job.Name + } + return djs, nil +} + +// MergeJobs - AFAICT this means "mark these jobs as free" +func (fj *freeJobService) MergeJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs dax.Jobs) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + err := dt.C.RawQuery("UPDATE jobs SET worker_id = NULL WHERE role = ? and database_id = ?", roleType, qdbid.DatabaseID).Exec() + return errors.Wrap(err, "marking jobs free") +} diff --git a/dax/controller/sqldb/freeworker.go b/dax/controller/sqldb/freeworker.go new file mode 100644 index 000000000..3ba26ddf3 --- /dev/null +++ b/dax/controller/sqldb/freeworker.go @@ -0,0 +1,96 @@ +package sqldb + +import ( + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/controller/balancer" + "github.com/featurebasedb/featurebase/v3/dax/models" + "github.com/featurebasedb/featurebase/v3/logger" + "github.com/pkg/errors" +) + +func NewFreeWorkerService(log logger.Logger) balancer.FreeWorkerService { + if log == nil { + log = logger.NopLogger + } + return &freeWorkerService{ + log: log, + } +} + +type freeWorkerService struct { + log logger.Logger +} + +func (fw *freeWorkerService) AddWorkers(tx dax.Transaction, roleType dax.RoleType, addrs ...dax.Address) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + workers := make(models.Workers, len(addrs)) + for i, addr := range addrs { + workers[i] = models.Worker{ + Address: addr, + Role: roleType, + } + } + + err := dt.C.Create(workers) + return errors.Wrap(err, "creating workers") +} + +func (fw *freeWorkerService) RemoveWorker(tx dax.Transaction, roleType dax.RoleType, addr dax.Address) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + err := dt.C.RawQuery("DELETE from workers where database_id is null and role = ? and address = ?", roleType, addr).Exec() + return errors.Wrap(err, "deleting") +} + +func (fw *freeWorkerService) PopWorkers(tx dax.Transaction, roleType dax.RoleType, num int) ([]dax.Address, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + results := make([]struct { + Address dax.Address `db:"address"` + }, 0, num) + err := dt.C.RawQuery("select address from workers where role = ? and database_id is NULL limit ?", roleType, num).All(&results) + if err != nil { + return nil, errors.Wrap(err, "querying") + } + if len(results) < num { + return nil, errors.Errorf("not enough free workers to get: wanted %d, have: %d", num, len(results)) + } + + ret := make([]dax.Address, num) + for i, res := range results { + ret[i] = res.Address + } + + return ret, nil +} + +func (fw *freeWorkerService) ListWorkers(tx dax.Transaction, roleType dax.RoleType) (dax.Addresses, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + workers := make(models.Workers, 0) + err := dt.C.Select("address").Where("role = ? and database_id is NULL", roleType).Order("address asc").All(&workers) + if err != nil { + return nil, errors.Wrap(err, "querying for workers") + } + + ret := make(dax.Addresses, len(workers)) + for i, w := range workers { + ret[i] = w.Address + } + + return ret, nil + +} diff --git a/dax/controller/sqldb/migrator.go b/dax/controller/sqldb/migrator.go new file mode 100644 index 000000000..d93598412 --- /dev/null +++ b/dax/controller/sqldb/migrator.go @@ -0,0 +1,96 @@ +package sqldb + +import ( + "fmt" + "io/fs" + "strings" + + "github.com/featurebasedb/featurebase/v3/logger" + "github.com/gobuffalo/pop/v6" +) + +// EmbedMigrator is a migrator for SQL and Fizz files which are +// embedded in any fs.FS. This is lifted directly from pop's +// FileMigrator and tweaked slightly to take an fs.FS instead of a +// file path. +type EmbedMigrator struct { + pop.Migrator + FS fs.FS + log logger.Logger +} + +// NewEmbedMigrator for a path and a Connection +func NewEmbedMigrator(fs fs.FS, c *pop.Connection, log logger.Logger) (*EmbedMigrator, error) { + fm := &EmbedMigrator{ + Migrator: pop.NewMigrator(c), + FS: fs, + log: log, + } + fm.SchemaPath = "" + + runner := func(mf pop.Migration, tx *pop.Connection) error { + f, err := fm.FS.Open(mf.Path) + if err != nil { + return err + } + defer f.Close() + content, err := pop.MigrationContent(mf, tx, f, true) + if err != nil { + return fmt.Errorf("error processing %s: %w", mf.Path, err) + } + if content == "" { + return nil + } + err = tx.RawQuery(content).Exec() + if err != nil { + return fmt.Errorf("error executing %s, sql: %s: %w", mf.Path, content, err) + } + return nil + } + + err := fm.findMigrations(runner) + if err != nil { + return fm, err + } + + return fm, nil +} + +func (fm *EmbedMigrator) findMigrations(runner func(mf pop.Migration, tx *pop.Connection) error) error { + return fs.WalkDir(fm.FS, "migrations", func(path string, d fs.DirEntry, err error) error { + fmt.Printf("walking path: %s, d: %v, err: %v\n", path, d, err) + if !d.IsDir() { + match, err := pop.ParseMigrationFilename(d.Name()) + if err != nil { + if strings.HasPrefix(err.Error(), "unsupported dialect") { + fm.log.Warnf("ignoring migration file with %s", err.Error()) + return nil + } + return err + } + if match == nil { + fm.log.Warnf("ignoring file %s because it does not match the migration file pattern", d.Name()) + return nil + } + mf := pop.Migration{ + Path: path, + Version: match.Version, + Name: match.Name, + DBType: match.DBType, + Direction: match.Direction, + Type: match.Type, + Runner: runner, + } + switch mf.Direction { + case "up": + fm.UpMigrations.Migrations = append(fm.UpMigrations.Migrations, mf) + case "down": + fm.DownMigrations.Migrations = append(fm.DownMigrations.Migrations, mf) + default: + // the regex only matches `(up|down)` for direction, so a panic here is appropriate + panic("got unknown migration direction " + mf.Direction) + } + } + return nil + }) +} diff --git a/dax/controller/sqldb/node.go b/dax/controller/sqldb/node.go new file mode 100644 index 000000000..276ac7664 --- /dev/null +++ b/dax/controller/sqldb/node.go @@ -0,0 +1,113 @@ +package sqldb + +import ( + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/controller" + "github.com/featurebasedb/featurebase/v3/dax/models" + "github.com/featurebasedb/featurebase/v3/logger" + "github.com/pkg/errors" +) + +var _ controller.NodeService = (*nodeService)(nil) + +func NewNodeService(log logger.Logger) *nodeService { + if log == nil { + log = logger.NopLogger + } + return &nodeService{ + log: log, + } +} + +type nodeService struct { + log logger.Logger +} + +func (n *nodeService) CreateNode(tx dax.Transaction, addr dax.Address, node *dax.Node) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + mnode := &models.Node{Address: node.Address} + err := dt.C.Create(mnode) + if err != nil { + return errors.Wrap(err, "creating node") + } + + nodeRoles := make(models.NodeRoles, len(node.RoleTypes)) + mnode.NodeRoles = nodeRoles + for i, rt := range node.RoleTypes { + mnode.NodeRoles[i] = models.NodeRole{ + NodeID: mnode.ID, + Role: rt, + } + } + err = dt.C.Create(&(mnode.NodeRoles)) + if err != nil { + return errors.Wrap(err, "creating node roles") + } + + return nil +} + +func (n *nodeService) ReadNode(tx dax.Transaction, addr dax.Address) (*dax.Node, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + node := &models.Node{} + err := dt.C.Eager().Where("address = ?", addr).First(node) + if err != nil { + return nil, errors.Wrap(err, "getting node") + } + + roleTypes := make([]dax.RoleType, len(node.NodeRoles)) + for i, nr := range node.NodeRoles { + roleTypes[i] = nr.Role + } + + return &dax.Node{ + Address: node.Address, + RoleTypes: roleTypes, + }, nil +} + +func (n *nodeService) DeleteNode(tx dax.Transaction, addr dax.Address) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + node := &models.Node{} + err := dt.C.Eager().Where("address = ?", addr).First(node) + if err != nil { + return errors.Wrap(err, "getting node") + } + err = dt.C.Destroy(node) + return errors.Wrap(err, "destroying node") +} + +func (n *nodeService) Nodes(tx dax.Transaction) ([]*dax.Node, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + nodes := []*models.Node{} + dt.C.Eager().Order("address asc").All(&nodes) + + ret := make([]*dax.Node, len(nodes)) + for i, node := range nodes { + ret[i] = &dax.Node{ + Address: node.Address, + RoleTypes: make([]dax.RoleType, len(node.NodeRoles)), + } + for j, nr := range node.NodeRoles { + ret[i].RoleTypes[j] = nr.Role + } + } + + return ret, nil +} diff --git a/dax/controller/sqldb/schemar.go b/dax/controller/sqldb/schemar.go new file mode 100644 index 000000000..af3f94e60 --- /dev/null +++ b/dax/controller/sqldb/schemar.go @@ -0,0 +1,458 @@ +package sqldb + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/pkg/errors" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/controller/schemar" + "github.com/featurebasedb/featurebase/v3/dax/models" + "github.com/featurebasedb/featurebase/v3/logger" +) + +func NewSchemar(log logger.Logger) schemar.Schemar { + if log == nil { + log = logger.NopLogger + } + return &Schemar{ + log: log, + } +} + +// Schemar is the SQL implementation of the schemar that stores all +// the information in a SQL database. +type Schemar struct { + log logger.Logger +} + +func (s *Schemar) CreateDatabase(tx dax.Transaction, qdb *dax.QualifiedDatabase) error { + // Ensure the database id is not blank. + if qdb.ID == "" { + return schemar.NewErrDatabaseIDInvalid(qdb.ID) + } + + // Ensure the database name is not blank. + if qdb.Name == "" { + return schemar.NewErrDatabaseNameInvalid(qdb.Name) + } + + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + if exists, err := dt.C.Where("id = ?", qdb.Database.ID).Exists(&models.Database{}); err != nil { + return errors.Wrap(err, "checking database existence") + } else if exists { + return dax.NewErrDatabaseIDExists(qdb.QualifiedID()) + } + + org := &models.Organization{ID: string(qdb.OrganizationID)} + if ok, err := dt.C.Where("id = ?", qdb.OrganizationID).Exists(org); err != nil { + return errors.Wrap(err, "checking for org") + } else if !ok { + if err := dt.C.Create(org); err != nil { + return errors.Wrap(err, "creating organization") + } + } + + db := toModelDatabase(qdb) + + if err := dt.C.Create(db); err != nil { + return errors.Wrap(err, "creating database object") + } + + return nil +} + +func (s *Schemar) DropDatabase(tx dax.Transaction, qdb dax.QualifiedDatabaseID) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + db := &models.Database{} + err := dt.C.RawQuery("DELETE from databases where id = ? RETURNING id", qdb.DatabaseID).First(db) + if isNoRowsError(err) { + return dax.NewErrDatabaseIDDoesNotExist(qdb) + } + + return errors.Wrap(err, "deleting database") +} + +func (s *Schemar) DatabaseByName(tx dax.Transaction, orgID dax.OrganizationID, dbname dax.DatabaseName) (*dax.QualifiedDatabase, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + db := &models.Database{} + err := dt.C.Where("organization_id = ? and name = ?", orgID, dbname).First(db) + if isNoRowsError(err) { + return nil, dax.NewErrDatabaseNameDoesNotExist(dbname) + } else if err != nil { + return nil, errors.Wrap(err, "finding database") + } + + return toQualifiedDatabase(db), nil +} + +func toModelDatabase(qdb *dax.QualifiedDatabase) *models.Database { + db := qdb.Database + return &models.Database{ + ID: string(db.ID), + Name: db.Name, + WorkersMin: db.Options.WorkersMin, + WorkersMax: db.Options.WorkersMax, + Description: db.Description, + Owner: db.Owner, + UpdatedBy: db.UpdatedBy, + // Tables: []*models.Table{}, + OrganizationID: string(qdb.OrganizationID), + // CreatedAt: time.Unix(db.CreatedAt, 0), + // UpdatedAt: time.Unix(db.UpdatedAt), + } +} + +func toQualifiedDatabase(db *models.Database) *dax.QualifiedDatabase { + return &dax.QualifiedDatabase{ + OrganizationID: dax.OrganizationID(db.OrganizationID), + Database: dax.Database{ + ID: dax.DatabaseID(db.ID), + Name: dax.DatabaseName(db.Name), + Options: dax.DatabaseOptions{ + WorkersMin: db.WorkersMin, + WorkersMax: db.WorkersMax, + }, + Description: db.Description, + Owner: db.Owner, + CreatedAt: db.CreatedAt.Unix(), // TODO is this right, or UnixNano, or...? + UpdatedAt: db.UpdatedAt.Unix(), + UpdatedBy: db.UpdatedBy, + }} +} + +func (s *Schemar) DatabaseByID(tx dax.Transaction, qdb dax.QualifiedDatabaseID) (*dax.QualifiedDatabase, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + db := &models.Database{} + err := dt.C.Find(db, string(qdb.DatabaseID)) + if isNoRowsError(err) { + return nil, dax.NewErrDatabaseIDDoesNotExist(qdb) + } else if err != nil { + return nil, errors.Wrap(err, "finding DB") + } + return toQualifiedDatabase(db), nil +} + +func (s *Schemar) SetDatabaseOption(tx dax.Transaction, qdbid dax.QualifiedDatabaseID, option string, value string) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + var val int64 + var err error + switch option { + case dax.DatabaseOptionWorkersMin: + option = "workers_min" // convert to table column name + val, err = strconv.ParseInt(value, 0, 64) + if err != nil { + return errors.Wrap(err, "parsing workers min value") + } + case dax.DatabaseOptionWorkersMax: + val, err = strconv.ParseInt(value, 0, 64) + option = "workers_max" // convert to table column name + if err != nil { + return errors.Wrap(err, "parsing workers max value") + } + default: + return errors.Errorf("unsupported database option: %s", option) + } + db := &models.Database{} + err = dt.C.RawQuery(fmt.Sprintf("UPDATE databases set %s = ? WHERE id = ? RETURNING id", option), val, qdbid.DatabaseID).First(db) + if isNoRowsError(err) { + return dax.NewErrDatabaseIDDoesNotExist(qdbid) + } else if err != nil { + return errors.Wrap(err, "updating option") + } + + return errors.Wrap(err, "updating database option") +} + +// Databases returns a list of databases. If the list of DatabaseIDs is +// empty, all databases will be returned. If greater than zero DatabaseIDs +// are passed in the second argument, only databases matching those IDs will +// be returned. +func (s *Schemar) Databases(tx dax.Transaction, orgID dax.OrganizationID, dbIDs ...dax.DatabaseID) ([]*dax.QualifiedDatabase, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + s.log.Debugf("Schemar: Databases: orgID: %s dbIDs: %v", orgID, dbIDs) + + dbs := []*models.Database{} + q := dt.C.Q() + if orgID != "" { + q = q.Where("organization_id = ?", orgID) + } + if len(dbIDs) > 0 { + ifaceIDs := make([]interface{}, len(dbIDs)) + for i, dbID := range dbIDs { + ifaceIDs[i] = dbID + } + q = q.Where("id in (?)", ifaceIDs...) + } + err := q.Order("created_at asc").All(&dbs) + if err != nil { + return nil, errors.Wrap(err, "finding databases") + } + + ret := make([]*dax.QualifiedDatabase, len(dbs)) + for i, db := range dbs { + ret[i] = toQualifiedDatabase(db) + } + s.log.Debugf("Schemar: Databases: returning %+v", ret) + + return ret, nil +} + +func (s *Schemar) CreateTable(tx dax.Transaction, qtbl *dax.QualifiedTable) error { + // Ensure the table id is not blank. + if qtbl.ID == "" { + return schemar.NewErrTableIDInvalid(qtbl.ID) + } + + // Ensure the table name is not blank. + if qtbl.Name == "" { + return schemar.NewErrTableNameInvalid(qtbl.Name) + } + + // Ensure that a primary key field is present and valid. + if !qtbl.HasValidPrimaryKey() { + return schemar.NewErrInvalidPrimaryKey() + } + + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + tbl := toModelTable(qtbl) + + err := dt.C.Eager().Create(tbl) + if isViolatesUniqueConstraint(err) { + return dax.NewErrTableIDExists(qtbl.QualifiedID()) + } + + return errors.Wrap(err, "creating database object") +} + +func toModelTable(qtbl *dax.QualifiedTable) *models.Table { + columns := make([]models.Column, len(qtbl.Fields)) + for i, fld := range qtbl.Fields { + columns[i] = toModelColumn(qtbl.Key(), fld) + } + return &models.Table{ + ID: string(qtbl.Key()), + Name: qtbl.Name, + Owner: qtbl.Owner, + OrganizationID: qtbl.OrganizationID, + Columns: columns, + UpdatedBy: qtbl.UpdatedBy, + DatabaseID: string(qtbl.QualifiedDatabaseID.DatabaseID), + Description: qtbl.Description, + PartitionN: qtbl.PartitionN, + } +} + +func toModelColumn(tk dax.TableKey, fld *dax.Field) models.Column { + optBytes, err := json.Marshal(fld.Options) + if err != nil { + panic(err) + } + return models.Column{ + Name: fld.Name, + Type: fld.Type, + TableID: string(tk), + Constraints: "TODO: unimplemented", + Options: string(optBytes), + } +} + +func toField(col models.Column) *dax.Field { + opts := dax.FieldOptions{} + err := json.Unmarshal([]byte(col.Options), &opts) + if err != nil { + panic(err) + } + return &dax.Field{ + Name: col.Name, + Type: col.Type, + Options: opts, + CreatedAt: col.CreatedAt.Unix(), + } +} + +func toQualifiedTable(mtbl *models.Table) *dax.QualifiedTable { + fields := make([]*dax.Field, len(mtbl.Columns)) + for i, col := range mtbl.Columns { + fields[i] = toField(col) + } + return &dax.QualifiedTable{ + QualifiedDatabaseID: dax.QualifiedDatabaseID{ + OrganizationID: mtbl.OrganizationID, + DatabaseID: dax.DatabaseID(mtbl.DatabaseID), + }, + Table: dax.Table{ + ID: dax.TableKey(mtbl.ID).QualifiedTableID().ID, + Name: mtbl.Name, + Fields: fields, + PartitionN: mtbl.PartitionN, + Description: mtbl.Description, + Owner: mtbl.Owner, + CreatedAt: mtbl.CreatedAt.Unix(), + UpdatedAt: mtbl.UpdatedAt.Unix(), + UpdatedBy: mtbl.UpdatedBy, + }, + } +} + +func (s *Schemar) DropTable(tx dax.Transaction, qtid dax.QualifiedTableID) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + err := dt.C.Destroy(&models.Table{ID: string(qtid.Key())}) + return errors.Wrap(err, "destroying table") +} + +func (s *Schemar) CreateField(tx dax.Transaction, qtid dax.QualifiedTableID, field *dax.Field) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + if field.Name == "" { + return schemar.NewErrFieldNameInvalid(field.Name) + } + + // we could probably make this a single query with an INSERT WHERE + // (subselect), but then would have to construct the whole insert + // by hand which would be annoying and error prone to keep up to + // date + cols := &models.Columns{} + err := dt.C.Where("name = ? and table_id = ?", field.Name, qtid.Key()).All(cols) + if err != nil { + return errors.Wrap(err, "looking up field") + } + if len(*cols) > 0 { + return dax.NewErrFieldExists(field.Name) + } + + col := toModelColumn(qtid.Key(), field) + err = dt.C.Create(&col) + return errors.Wrap(err, "creating column") +} + +func (s *Schemar) DropField(tx dax.Transaction, qtid dax.QualifiedTableID, fieldName dax.FieldName) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + col := &models.Column{} + err := dt.C.Where("table_id = ? and name = ?", qtid.Key(), fieldName).First(col) + if err != nil { + if isNoRowsError(err) { + return dax.NewErrFieldDoesNotExist(fieldName) + } + + return errors.Wrap(err, "querying for field") + } + + err = dt.C.Destroy(col) + + return errors.Wrap(err, "destroying col") +} + +func (s *Schemar) Table(tx dax.Transaction, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + tbl := &models.Table{} + if qtid.ID != "" { + if err := dt.C.Eager().Find(tbl, qtid.Key()); err != nil { + if isNoRowsError(err) { + return nil, dax.NewErrTableIDDoesNotExist(qtid) + } + return nil, errors.Wrap(err, "finding table by ID") + } + } else { + if err := dt.C.Eager().Where("database_id = ? and name = ?", qtid.DatabaseID, qtid.Name).First(tbl); err != nil { + if isNoRowsError(err) { + return nil, dax.NewErrTableNameDoesNotExist(qtid.Name) + } + return nil, errors.Wrap(err, "finding table by name") + } + } + + return toQualifiedTable(tbl), nil +} + +// Tables returns a list of tables in the given database. If tableIDs +// are given, only tables with matching IDs are returned. +func (s *Schemar) Tables(tx dax.Transaction, qdbid dax.QualifiedDatabaseID, tableIDs ...dax.TableID) ([]*dax.QualifiedTable, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + query := dt.C.Where("database_id = ?", qdbid.DatabaseID) + if len(tableIDs) > 0 { + ifaceIDs := make([]interface{}, len(tableIDs)) + for i, tableID := range tableIDs { + ifaceIDs[i] = dax.QualifiedTableID{QualifiedDatabaseID: qdbid, ID: tableID}.Key() + } + query = query.Where("id in (?)", ifaceIDs) + } + tables := []*models.Table{} + err := query.Eager().Order("created_at asc").All(&tables) + if err != nil { + return nil, errors.Wrap(err, "querying all tables") + } + + ret := make([]*dax.QualifiedTable, len(tables)) + for i, tab := range tables { + ret[i] = toQualifiedTable(tab) + } + + return ret, nil +} + +// TableID is a reverse-lookup method to get the TableID for a given +// qualified TableName. +func (s *Schemar) TableID(tx dax.Transaction, qdbid dax.QualifiedDatabaseID, tableName dax.TableName) (dax.QualifiedTableID, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.QualifiedTableID{}, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + tbl := &models.Table{} + if err := dt.C.Where("database_id = ? and name = ?", qdbid.DatabaseID, tableName).First(tbl); err != nil { + if isNoRowsError(err) { + return dax.QualifiedTableID{}, dax.NewErrTableNameDoesNotExist(tableName) + } + return dax.QualifiedTableID{}, errors.Wrapf(err, "looking up table by name '%s', dbid: '%s'", tableName, qdbid.DatabaseID) + } + + return dax.TableKey(tbl.ID).QualifiedTableID(), nil +} diff --git a/dax/controller/sqldb/test.go b/dax/controller/sqldb/test.go new file mode 100644 index 000000000..cf1a78eb7 --- /dev/null +++ b/dax/controller/sqldb/test.go @@ -0,0 +1,41 @@ +package sqldb + +import ( + "fmt" + "math/rand" + "os" + "time" + + "github.com/featurebasedb/featurebase/v3/dax/controller" +) + +func EnvOr(envName, defaultVal string) string { + val, ok := os.LookupEnv(envName) + if !ok { + return defaultVal + } + return val +} + +func GetTestConfig() *controller.SQLDBConfig { + return &controller.SQLDBConfig{ + Dialect: "postgres", + Database: EnvOr("SQLDB_DB", "dax_test"), + Host: EnvOr("SQLDB_HOST", "127.0.0.1"), + Port: EnvOr("SQLDB_PORT", "5432"), + User: EnvOr("SQLDB_USER", "postgres"), + Password: EnvOr("SQLDB_PASSWORD", "testpass"), + } +} + +func GetTestConfigRandomDB(dbprefix string) *controller.SQLDBConfig { + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + return &controller.SQLDBConfig{ + Dialect: "postgres", + Database: fmt.Sprintf("%s_%d", dbprefix, rnd.Int()), + Host: EnvOr("SQLDB_HOST", "127.0.0.1"), + Port: EnvOr("SQLDB_PORT", "5432"), + User: EnvOr("SQLDB_USER", "postgres"), + Password: EnvOr("SQLDB_PASSWORD", "testpass"), + } +} diff --git a/dax/controller/sqldb/transactor.go b/dax/controller/sqldb/transactor.go new file mode 100644 index 000000000..ad1919d50 --- /dev/null +++ b/dax/controller/sqldb/transactor.go @@ -0,0 +1,46 @@ +package sqldb + +import ( + "context" + + "database/sql" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/errors" + "github.com/gobuffalo/pop/v6" +) + +// Transactor wraps a pop Connection to make it into a dax.Transactor +// which can be used by the controller agnostic of implementation. +type Transactor struct { + *pop.Connection +} + +func (t Transactor) BeginTx(ctx context.Context, writable bool) (dax.Transaction, error) { + cn, err := t.NewTransactionContextOptions(ctx, &sql.TxOptions{ReadOnly: !writable}) + if err != nil { + return nil, errors.Wrap(err, "getting SQL transaction") + } + return &DaxTransaction{C: cn}, nil +} + +func (t Transactor) Close() error { + return t.Connection.Close() +} + +// DaxTransaction is a thin wrapper to create a dax.Transaction from a +// pop Transaction/Connection. +type DaxTransaction struct { + C *pop.Connection +} + +func (w *DaxTransaction) Commit() error { + return w.C.TX.Commit() +} + +func (w *DaxTransaction) Context() context.Context { + return w.C.Context() +} +func (w *DaxTransaction) Rollback() error { + return w.C.TX.Rollback() +} diff --git a/dax/controller/sqldb/util.go b/dax/controller/sqldb/util.go new file mode 100644 index 000000000..d640ca562 --- /dev/null +++ b/dax/controller/sqldb/util.go @@ -0,0 +1,11 @@ +package sqldb + +import "strings" + +func isNoRowsError(err error) bool { + return err != nil && strings.Contains(err.Error(), "no rows in result set") +} + +func isViolatesUniqueConstraint(err error) bool { + return err != nil && strings.Contains(err.Error(), "duplicate key value violates unique constraint") +} diff --git a/dax/controller/sqldb/workerjob.go b/dax/controller/sqldb/workerjob.go new file mode 100644 index 000000000..8174aeeb7 --- /dev/null +++ b/dax/controller/sqldb/workerjob.go @@ -0,0 +1,300 @@ +package sqldb + +import ( + "fmt" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/controller/balancer" + "github.com/featurebasedb/featurebase/v3/dax/models" + "github.com/featurebasedb/featurebase/v3/logger" + "github.com/gofrs/uuid" + "github.com/pkg/errors" +) + +func NewWorkerJobService(log logger.Logger) balancer.WorkerJobService { + if log == nil { + log = logger.NopLogger + } + return &workerJobService{ + log: log, + } +} + +type workerJobService struct { + log logger.Logger +} + +func (w *workerJobService) WorkersJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) ([]dax.WorkerInfo, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + workers := models.Workers{} + err := dt.C.Eager().Where("role = ? and database_id = ?", roleType, qdbid.DatabaseID).Order("address asc").All(&workers) + if err != nil { + return nil, errors.Wrap(err, "getting workers") + } + + ret := make([]dax.WorkerInfo, len(workers)) + for i, worker := range workers { + ret[i].Address = worker.Address + ret[i].Jobs = make([]dax.Job, len(worker.Jobs)) + for j, job := range worker.Jobs { + ret[i].Jobs[j] = job.Name + } + } + + return ret, nil +} + +func (w *workerJobService) WorkerCount(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (int, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return 0, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + worker := &models.Worker{} + cnt, err := dt.C.Where("role = ? and database_id = ?", roleType, qdbid.DatabaseID).Count(worker) + return cnt, errors.Wrap(err, "getting count") +} + +func (w *workerJobService) ListWorkers(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Addresses, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + workers := models.Workers{} + err := dt.C.Select("address").Where("role = ? and database_id = ?", roleType, qdbid.DatabaseID).Order("address asc").All(&workers) + if err != nil { + return nil, errors.Wrap(err, "getting workers") + } + + ret := make(dax.Addresses, len(workers)) + for i, wrkr := range workers { + ret[i] = wrkr.Address + } + + return ret, nil +} + +func (w *workerJobService) CreateWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + worker := &models.Worker{} + err := dt.C.RawQuery("UPDATE workers SET database_id = ? WHERE role = ? and address = ? RETURNING workers.ID", qdbid.DatabaseID, roleType, addr).First(worker) + + return errors.Wrap(err, "associating worker to database") +} + +func (w *workerJobService) FreeWorkers(tx dax.Transaction, addrs ...dax.Address) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + if len(addrs) == 0 { + return nil + } + + err := dt.C.RawQuery("UPDATE workers set database_id = NULL where address in (?)", addrs).Exec() + return errors.Wrap(err, "updating workers") +} + +func (w *workerJobService) DeleteWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + worker := &models.Worker{} + err := dt.C.Where("address = ? and role = ? and database_id = ?", addr, roleType, qdbid.DatabaseID).First(worker) + if err != nil { + return errors.Wrap(err, "getting worker") + } + err = dt.C.Destroy(worker) + return errors.Wrap(err, "deleting worker") +} + +func (w *workerJobService) CreateJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, job ...dax.Job) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + worker := &models.Worker{} + err := dt.C.Where("address = ? and role = ?", addr, roleType).First(worker) + if err != nil { + return errors.Wrap(err, "getting worker") + } + + jobs := models.Jobs{} + err = dt.C.RawQuery("UPDATE jobs SET worker_id = ? WHERE role = ? and name in (?) RETURNING jobs.ID, jobs.Name", worker.ID, roleType, job).All(&jobs) + if err != nil { + return errors.Wrap(err, "updating jobs") + } + + // create jobs not in "jobs" + toCreate := jobsNotUpdated(job, jobs, worker) + + err = dt.C.Create(toCreate) + if err != nil { + return errors.Wrap(err, "creating jobs") + } + + return errors.Wrap(err, "creating jobs") +} + +func jobsNotUpdated(incomingJobs []dax.Job, created models.Jobs, worker *models.Worker) (toCreate models.Jobs) { +outer: + for _, incJob := range incomingJobs { + for _, createdJob := range created { + if createdJob.Name == incJob { + continue outer + } + } + toCreate = append(toCreate, + models.Job{ + Name: incJob, + Role: worker.Role, + DatabaseID: dax.DatabaseID(worker.DatabaseID.String), + Worker: worker, + }, + ) + } + return toCreate +} + +func (w *workerJobService) DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, job dax.Job) error { + dt, ok := tx.(*DaxTransaction) + if !ok { + return dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + worker := &models.Worker{} + err := dt.C.Select("id").Where("role = ? and database_id = ? and address = ?", roleType, qdbid.DatabaseID, addr).First(worker) + if err != nil { + return errors.Wrap(err, "getting worker") + } + + jerb := &models.Job{} + dt.C.Select("id").Where("role = ? and worker_id = ? and name = ?", roleType, worker.ID, job).First(jerb) + if err != nil { + return errors.Wrap(err, "getting job") + } + + err = dt.C.Destroy(jerb) + if err != nil { + return errors.Wrap(err, "destroying job") + } + + return nil +} + +func (w *workerJobService) DeleteJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) (balancer.InternalDiffs, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + results := []struct { + ID uuid.UUID `db:"id"` + Name dax.Job `db:"name"` + Address dax.Address `db:"address"` + }{} + err := dt.C.RawQuery("select j.id, j.name, w.address from jobs j inner join workers w on j.worker_id = w.id where j.role = ? and j.database_id = ? and j.name LIKE ?", roleType, qtid.QualifiedDatabaseID.DatabaseID, fmt.Sprintf("%s%%", qtid.Key())).All(&results) + if err != nil { + return nil, errors.Wrap(err, "querying for jobs") + } + + idiffs := make(balancer.InternalDiffs) + ids := make([]uuid.UUID, 0, len(results)) + for _, job := range results { + idiffs.Removed(job.Address, job.Name) + ids = append(ids, job.ID) + } + + if len(ids) > 0 { + err = dt.C.RawQuery("DELETE FROM jobs WHERE id in (?)", ids).Exec() + } + + return idiffs, errors.Wrap(err, "deleting jobs") +} + +func (w *workerJobService) JobCounts(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr ...dax.Address) (map[dax.Address]int, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + results := []struct { + Address dax.Address `db:"address"` + Count int `db:"count"` + }{} + var err error + if len(addr) == 0 { + qstring := `select address, count(*) as count + from workers w inner join jobs j on j.worker_id = w.id + where w.database_id = ? and w.role = ? + group by w.address` + err = dt.C.RawQuery(qstring, qdbid.DatabaseID, roleType).All(&results) + } else { + qstring := `select address, count(*) as count + from workers w inner join jobs j on j.worker_id = w.id + where w.address in (?) and w.database_id = ? and w.role = ? + group by w.address` + err = dt.C.RawQuery(qstring, addr, qdbid.DatabaseID, roleType).All(&results) + } + if err != nil { + return nil, errors.Wrap(err, "querying for jobs") + } + ret := make(map[dax.Address]int) + for _, res := range results { + ret[res.Address] = res.Count + } + + return ret, nil +} + +func (w *workerJobService) ListJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) (dax.Jobs, error) { + dt, ok := tx.(*DaxTransaction) + if !ok { + return nil, dax.NewErrInvalidTransaction("*sqldb.DaxTransaction") + } + + worker := &models.Worker{} + err := dt.C.Eager().Where("role = ? and database_id = ? and address = ?", roleType, qdbid.DatabaseID, addr).First(worker) + if isNoRowsError(err) { + return nil, nil + } else if err != nil { + return nil, errors.Wrap(err, "getting worker") + } + + ret := make(dax.Jobs, len(worker.Jobs)) + // jobs are ordered by "name asc" defined on the worker model. + for i, job := range worker.Jobs { + ret[i] = job.Name + } + + return ret, nil +} + +func (w *workerJobService) DatabaseForWorker(tx dax.Transaction, addr dax.Address) dax.DatabaseKey { + dt, ok := tx.(*DaxTransaction) + if !ok { + panic("wrong transaction type passed to sqldb DatabaseForWorker") + } + + db := &models.Database{} + err := dt.C.RawQuery("select d.ID, d.organization_id from databases d inner join workers w on d.id = w.database_id where w.address = ?", addr).First(db) + if isNoRowsError(err) { + return "" + } else if err != nil { + panic(err) + } + + return dax.QualifiedDatabase{OrganizationID: dax.OrganizationID(db.OrganizationID), Database: dax.Database{ID: dax.DatabaseID(db.ID)}}.Key() +} diff --git a/dax/controller/sqldb/workerjob_test.go b/dax/controller/sqldb/workerjob_test.go new file mode 100644 index 000000000..dd270ccb0 --- /dev/null +++ b/dax/controller/sqldb/workerjob_test.go @@ -0,0 +1,40 @@ +package sqldb + +import ( + "testing" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/models" + "github.com/gobuffalo/nulls" + "github.com/gofrs/uuid" + "github.com/stretchr/testify/require" +) + +func TestJobsNotUpdated(t *testing.T) { + u, _ := uuid.NewV4() + u2, _ := uuid.NewV4() + incJobs := []dax.Job{"job1", "job2", "job3", "job4"} + created := models.Jobs{ + models.Job{ + ID: u, + Name: "job2", + }, + } + + toCreate := jobsNotUpdated(incJobs, created, &models.Worker{ + ID: u2, + Role: "compute", + DatabaseID: nulls.NewString("dbid"), + }) + + require.Equal(t, 3, len(toCreate)) + + // Test when 0 jobs are updated + toCreate = jobsNotUpdated(incJobs, models.Jobs{}, &models.Worker{ + ID: u2, + Role: "compute", + DatabaseID: nulls.NewString("dbid"), + }) + + require.Equal(t, 4, len(toCreate)) +} diff --git a/dax/controller/transactor.go b/dax/controller/transactor.go new file mode 100644 index 000000000..9922ac750 --- /dev/null +++ b/dax/controller/transactor.go @@ -0,0 +1,12 @@ +package controller + +import ( + "context" + + "github.com/featurebasedb/featurebase/v3/dax" +) + +type Transactor interface { + BeginTx(ctx context.Context, writable bool) (dax.Transaction, error) + Close() error +} diff --git a/dax/directive_version_test.go b/dax/directive_version_test.go new file mode 100644 index 000000000..80cd1f854 --- /dev/null +++ b/dax/directive_version_test.go @@ -0,0 +1,31 @@ +package dax_test + +import ( + "context" + "testing" + + "github.com/featurebasedb/featurebase/v3/dax/controller/sqldb" + "github.com/featurebasedb/featurebase/v3/logger" + "github.com/stretchr/testify/require" +) + +func TestDirectiveVersion(t *testing.T) { + trans, err := sqldb.Connect(sqldb.GetTestConfigRandomDB("directive_version"), logger.StderrLogger) // TODO running migrations takes kind of a long time, consolidate w/ other SQL tests + require.NoError(t, err, "connecting") + + tx, err := trans.BeginTx(context.Background(), true) + require.NoError(t, err, "getting transaction") + + defer func() { + err := tx.Rollback() + if err != nil { + t.Logf("rolling back: %v", err) + } + }() + + dvSvc := sqldb.NewDirectiveVersion(nil) + + n, err := dvSvc.Increment(tx, 1) + require.NoError(t, err) + require.Equal(t, uint64(2), n) +} diff --git a/dax/errors.go b/dax/errors.go index a01c6a148..eb653fbb2 100644 --- a/dax/errors.go +++ b/dax/errors.go @@ -115,9 +115,9 @@ func NewErrFieldExists(fieldName FieldName) error { ) } -func NewErrInvalidTransaction() error { +func NewErrInvalidTransaction(txType string) error { return errors.New( ErrInvalidTransaction, - "tx is not a *boltdb.Tx", + fmt.Sprintf("tx is not expected type: '%s'", txType), ) } diff --git a/dax/migrations.go b/dax/migrations.go new file mode 100644 index 000000000..6665abf0a --- /dev/null +++ b/dax/migrations.go @@ -0,0 +1,9 @@ +package dax + +import "embed" + +// MigrationsFS will hold the contents of the migrations directory as +// a filesystem object embedded in the binary. Pretty neat! +// +//go:embed migrations/* +var MigrationsFS embed.FS diff --git a/dax/migrations/001_initial.down.fizz b/dax/migrations/001_initial.down.fizz new file mode 100644 index 000000000..53690d78c --- /dev/null +++ b/dax/migrations/001_initial.down.fizz @@ -0,0 +1,4 @@ +drop_table("columns") +drop_table("tables") +drop_table("databases") +drop_table("organizations") \ No newline at end of file diff --git a/dax/migrations/001_initial.up.fizz b/dax/migrations/001_initial.up.fizz new file mode 100644 index 000000000..f47c6ad56 --- /dev/null +++ b/dax/migrations/001_initial.up.fizz @@ -0,0 +1,84 @@ + +create_table("organizations") { + t.Column("id", "string", {primary: true}) + t.Timestamps() +} + + +create_table("databases") { + t.Column("id", "string", {primary: true}) + t.Column("organization_id", "string") + t.Column("description", "string") + t.Column("owner", "string") + t.Column("updated_by", "string") + t.Column("workers_min", "int") + t.Column("workers_max", "int") + t.Column("name", "string") + t.ForeignKey("organization_id", {"organizations": ["id"]}, {"on_delete": "cascade"}) + t.Timestamps() +} + +create_table("tables") { + t.Column("id", "string", {primary: true}) + t.Column("name", "string", {}) + t.Column("owner", "string", {}) + t.Column("updated_by", "string", {}) + t.Column("description", "string", {}) + t.Column("partition_n", "integer", {}) + t.Column("database_id", "string") + t.Column("organization_id", "string") + t.ForeignKey("database_id", {"databases": ["id"]}, {"on_delete": "cascade"}) + t.Timestamps() +} + +create_table("columns") { + t.Column("id", "uuid", {primary: true}) + t.Column("name", "string", {}) + t.Column("type", "string", {}) + t.Column("constraints", "string", {}) + t.Column("options", "string", {}) + t.Column("table_id", "string") + t.ForeignKey("table_id", {"tables": ["id"]}, {"on_delete": "cascade"}) + t.Timestamps() +} + +create_table("nodes") { + t.Column("id", "uuid", {primary: true}) + t.Column("address", "string") + t.Timestamps() +} + +create_table("node_roles") { + t.Column("id", "uuid", {primary: true}) + t.Column("node_id", "uuid") + t.Column("role", "string") + t.ForeignKey("node_id", {"nodes": ["id"]}, {"on_delete": "cascade"}) + t.Timestamps() +} + +create_table("workers") { + t.Column("id", "uuid", {primary: true}) + t.Column("address", "string") + t.Column("role", "string") + t.Column("database_id", "string", {"null": true}) + t.ForeignKey("database_id", {"databases": ["id"]}, {"null": true}) +} + +create_table("jobs") { + t.Column("id", "uuid", {primary: true}) + t.Column("name", "string") + t.Column("role", "string") + t.Column("worker_id", "uuid", {"null": true}) + t.ForeignKey("worker_id", {"workers": ["id"]}, {"null": true}) + t.Column("database_id", "string") + t.ForeignKey("database_id", {"databases": ["id"]}, {"on_delete": "cascade"}) + t.Timestamps() +} + +create_table("directive_versions") { + t.Column("id", "int", {primary: true}) + t.Column("version", "int") + t.Timestamps() +} + +sql("INSERT INTO directive_versions (id, version, created_at, updated_at) VALUES (1, 1, '1970-01-01T00:00', '1970-01-01T00:00');") diff --git a/dax/models/column.go b/dax/models/column.go new file mode 100644 index 000000000..d0799dbe1 --- /dev/null +++ b/dax/models/column.go @@ -0,0 +1,61 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/gobuffalo/pop/v6" + "github.com/gobuffalo/validate/v3" + "github.com/gobuffalo/validate/v3/validators" + "github.com/gofrs/uuid" +) + +// Column is used by pop to map your columns database table to your go code. +type Column struct { + ID uuid.UUID `json:"id" db:"id"` + Name dax.FieldName `json:"name" db:"name"` + Type dax.BaseType `json:"type" db:"type"` + TableID string `json:"table_id" db:"table_id"` + Constraints string `json:"constraints" db:"constraints"` + Options string `json:"options" db:"options"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} + +// String is not required by pop and may be deleted +func (c *Column) String() string { + jc, _ := json.MarshalIndent(c, " ", " ") //nolint:errchkjson + return string(jc) +} + +// Columns is not required by pop and may be deleted +type Columns []Column + +// String is not required by pop and may be deleted +func (c Columns) String() string { + jc, _ := json.MarshalIndent(c, " ", " ") //nolint:errchkjson + return string(jc) +} + +// Validate gets run every time you call a "pop.Validate*" (pop.ValidateAndSave, pop.ValidateAndCreate, pop.ValidateAndUpdate) method. +// This method is not required and may be deleted. +func (c *Column) Validate(tx *pop.Connection) (*validate.Errors, error) { + return validate.Validate( + &validators.StringIsPresent{Field: string(c.Name), Name: "Name"}, + &validators.StringIsPresent{Field: string(c.Type), Name: "Type"}, + &validators.StringIsPresent{Field: c.Constraints, Name: "Constraints"}, + ), nil +} + +// ValidateCreate gets run every time you call "pop.ValidateAndCreate" method. +// This method is not required and may be deleted. +func (c *Column) ValidateCreate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} + +// ValidateUpdate gets run every time you call "pop.ValidateAndUpdate" method. +// This method is not required and may be deleted. +func (c *Column) ValidateUpdate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} diff --git a/dax/models/database.go b/dax/models/database.go new file mode 100644 index 000000000..53feed82a --- /dev/null +++ b/dax/models/database.go @@ -0,0 +1,60 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/gobuffalo/pop/v6" + "github.com/gobuffalo/validate/v3" +) + +// Database is used by pop to map your databases database table to your go code. +type Database struct { + // should be DatabaseID, but pop doesn't allow ID to be a custom type + ID string `json:"id" db:"id"` + Name dax.DatabaseName `json:"name" db:"name"` + WorkersMin int `json:"workers_min" db:"workers_min"` + WorkersMax int `json:"workers_max" db:"workers_max"` + Description string `json:"description" db:"description"` + Owner string `json:"owner" db:"owner"` + UpdatedBy string `json:"updated_by" db:"updated_by"` + Tables Tables `json:"tables" has_many:"tables"` + Organization *Organization `json:"organization" belongs_to:"organization"` + OrganizationID string `json:"organization_id" db:"organization_id"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} + +// String is not required by pop and may be deleted +func (d Database) String() string { + jd, _ := json.MarshalIndent(d, " ", " ") //nolint:errchkjson + return string(jd) +} + +// Databases is not required by pop and may be deleted +type Databases []Database + +// String is not required by pop and may be deleted +func (d Databases) String() string { + jd, _ := json.Marshal(d) //nolint:errchkjson + return string(jd) +} + +// Validate gets run every time you call a "pop.Validate*" (pop.ValidateAndSave, pop.ValidateAndCreate, pop.ValidateAndUpdate) method. +// This method is not required and may be deleted. +func (d *Database) Validate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} + +// ValidateCreate gets run every time you call "pop.ValidateAndCreate" method. +// This method is not required and may be deleted. +func (d *Database) ValidateCreate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} + +// ValidateUpdate gets run every time you call "pop.ValidateAndUpdate" method. +// This method is not required and may be deleted. +func (d *Database) ValidateUpdate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} diff --git a/dax/models/directiveversion.go b/dax/models/directiveversion.go new file mode 100644 index 000000000..218c3947a --- /dev/null +++ b/dax/models/directiveversion.go @@ -0,0 +1,20 @@ +package models + +import ( + "encoding/json" + "time" +) + +// DirectiveVersion holds what version the current directive is +type DirectiveVersion struct { + ID int `json:"id" db:"id"` + Version int `json:"version" db:"version"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} + +// String is not required by pop and may be deleted +func (t *DirectiveVersion) String() string { + jt, _ := json.MarshalIndent(t, " ", " ") //nolint:errchkjson + return string(jt) +} diff --git a/dax/models/job.go b/dax/models/job.go new file mode 100644 index 000000000..fd8bac483 --- /dev/null +++ b/dax/models/job.go @@ -0,0 +1,60 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/gobuffalo/nulls" + "github.com/gobuffalo/pop/v6" + "github.com/gobuffalo/validate/v3" + "github.com/gobuffalo/validate/v3/validators" + "github.com/gofrs/uuid" +) + +// Job represents a job which can be assigned to a worker or free (unassigned). +type Job struct { + ID uuid.UUID `json:"id" db:"id"` + Name dax.Job `json:"name" db:"name"` + Role dax.RoleType `json:"role" db:"role"` + DatabaseID dax.DatabaseID `json:"database_id" db:"database_id"` + WorkerID nulls.UUID `json:"-" db:"worker_id"` + Worker *Worker `json:"worker" db:"-" belongs_to:"worker"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} + +// String is not required by pop and may be deleted +func (t *Job) String() string { + jt, _ := json.MarshalIndent(t, " ", " ") //nolint:errchkjson + return string(jt) +} + +// Jobs is not required by pop and may be deleted +type Jobs []Job + +// String is not required by pop and may be deleted +func (t Jobs) String() string { + jt, _ := json.MarshalIndent(t, " ", " ") //nolint:errchkjson + return string(jt) +} + +// Validate gets run every time you call a "pop.Validate*" (pop.ValidateAndSave, pop.ValidateAndCreate, pop.ValidateAndUpdate) method. +// This method is not required and may be deleted. +func (t *Job) Validate(tx *pop.Connection) (*validate.Errors, error) { + return validate.Validate( + &validators.StringIsPresent{Field: string(t.Name), Name: "Name"}, + ), nil +} + +// ValidateCreate gets run every time you call "pop.ValidateAndCreate" method. +// This method is not required and may be deleted. +func (t *Job) ValidateCreate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} + +// ValidateUpdate gets run every time you call "pop.ValidateAndUpdate" method. +// This method is not required and may be deleted. +func (t *Job) ValidateUpdate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} diff --git a/dax/models/node.go b/dax/models/node.go new file mode 100644 index 000000000..9a387a277 --- /dev/null +++ b/dax/models/node.go @@ -0,0 +1,56 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/gobuffalo/pop/v6" + "github.com/gobuffalo/validate/v3" + "github.com/gobuffalo/validate/v3/validators" + "github.com/gofrs/uuid" +) + +// Node represents a host or server that is available to work on jobs. +type Node struct { + ID uuid.UUID `json:"id" db:"id"` + Address dax.Address `json:"address" db:"address"` + NodeRoles NodeRoles `json:"node_roles" has_many:"node_roles"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} + +// String is not required by pop and may be deleted +func (t *Node) String() string { + jt, _ := json.MarshalIndent(t, " ", " ") //nolint:errchkjson + return string(jt) +} + +// Nodes is not required by pop and may be deleted +type Nodes []*Node + +// String is not required by pop and may be deleted +func (t Nodes) String() string { + jt, _ := json.MarshalIndent(t, " ", " ") //nolint:errchkjson + return string(jt) +} + +// Validate gets run every time you call a "pop.Validate*" (pop.ValidateAndSave, pop.ValidateAndCreate, pop.ValidateAndUpdate) method. +// This method is not required and may be deleted. +func (t *Node) Validate(tx *pop.Connection) (*validate.Errors, error) { + return validate.Validate( + &validators.StringIsPresent{Field: string(t.Address), Name: "Address"}, + ), nil +} + +// ValidateCreate gets run every time you call "pop.ValidateAndCreate" method. +// This method is not required and may be deleted. +func (t *Node) ValidateCreate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} + +// ValidateUpdate gets run every time you call "pop.ValidateAndUpdate" method. +// This method is not required and may be deleted. +func (t *Node) ValidateUpdate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} diff --git a/dax/models/node_roles.go b/dax/models/node_roles.go new file mode 100644 index 000000000..558e2940e --- /dev/null +++ b/dax/models/node_roles.go @@ -0,0 +1,31 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/gofrs/uuid" +) + +// NodeRole holds information about what types of jobs (roles) each node can perform. +type NodeRole struct { + ID uuid.UUID `json:"id" db:"id"` + NodeID uuid.UUID `json:"node_id" db:"node_id"` + Role dax.RoleType `json:"role" db:"role"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} + +// String is not required by pop and may be deleted +func (t *NodeRole) String() string { + jt, _ := json.MarshalIndent(t, " ", " ") //nolint:errchkjson + return string(jt) +} + +type NodeRoles []NodeRole + +func (t NodeRoles) String() string { + jt, _ := json.MarshalIndent(t, " ", " ") //nolint:errchkjson + return string(jt) +} diff --git a/dax/models/organization.go b/dax/models/organization.go new file mode 100644 index 000000000..8bef1a5fe --- /dev/null +++ b/dax/models/organization.go @@ -0,0 +1,51 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/gobuffalo/pop/v6" + "github.com/gobuffalo/validate/v3" +) + +// Organization is used by pop to map your organizations database table to your go code. +type Organization struct { + // Would love for this to be OrganizationID, but pop doesn't allow it for primary keys + ID string `json:"id" db:"id"` + Databases Databases `json:"databases" db:"-" has_many:"databases" order_by:"created_at asc"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} + +// String is not required by pop and may be deleted +func (o *Organization) String() string { + jo, _ := json.MarshalIndent(o, " ", " ") //nolint:errchkjson + return string(jo) +} + +// Organizations is not required by pop and may be deleted +type Organizations []Organization + +// String is not required by pop and may be deleted +func (o Organizations) String() string { + jo, _ := json.MarshalIndent(o, " ", " ") //nolint:errchkjson + return string(jo) +} + +// Validate gets run every time you call a "pop.Validate*" (pop.ValidateAndSave, pop.ValidateAndCreate, pop.ValidateAndUpdate) method. +// This method is not required and may be deleted. +func (o *Organization) Validate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} + +// ValidateCreate gets run every time you call "pop.ValidateAndCreate" method. +// This method is not required and may be deleted. +func (o *Organization) ValidateCreate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} + +// ValidateUpdate gets run every time you call "pop.ValidateAndUpdate" method. +// This method is not required and may be deleted. +func (o *Organization) ValidateUpdate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} diff --git a/dax/models/table.go b/dax/models/table.go new file mode 100644 index 000000000..5a2a526b8 --- /dev/null +++ b/dax/models/table.go @@ -0,0 +1,67 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/gobuffalo/pop/v6" + "github.com/gobuffalo/validate/v3" + "github.com/gobuffalo/validate/v3/validators" +) + +// Table is used by pop to map your tables database table to your go code. +type Table struct { + // ID will store the dax.Table.Key(), but must be string type due to pop's nonsense + ID string `json:"id" db:"id"` + Name dax.TableName `json:"name" db:"name"` + Owner string `json:"owner" db:"owner"` + UpdatedBy string `json:"updated_by" db:"updated_by"` + Database *Database `json:"database" belongs_to:"database"` + DatabaseID string `json:"database_id" db:"database_id"` + OrganizationID dax.OrganizationID `json:"organization_id" db:"organization_id"` + Description string `json:"description" db:"description"` + PartitionN int `json:"partition_n" db:"partition_n"` + Columns Columns `json:"columns" has_many:"columns" order_by:"created_at asc"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` +} + +// String is not required by pop and may be deleted +func (t *Table) String() string { + jt, _ := json.MarshalIndent(t, " ", " ") //nolint:errchkjson + return string(jt) +} + +// Tables is not required by pop and may be deleted +type Tables []*Table + +// String is not required by pop and may be deleted +func (t Tables) String() string { + jt, _ := json.MarshalIndent(t, " ", " ") //nolint:errchkjson + return string(jt) +} + +// Validate gets run every time you call a "pop.Validate*" (pop.ValidateAndSave, pop.ValidateAndCreate, pop.ValidateAndUpdate) method. +// This method is not required and may be deleted. +func (t *Table) Validate(tx *pop.Connection) (*validate.Errors, error) { + return validate.Validate( + &validators.StringIsPresent{Field: string(t.Name), Name: "Name"}, + &validators.StringIsPresent{Field: t.Owner, Name: "Owner"}, + &validators.StringIsPresent{Field: t.UpdatedBy, Name: "UpdatedBy"}, + &validators.StringIsPresent{Field: t.Description, Name: "Description"}, + &validators.IntIsPresent{Field: t.PartitionN, Name: "PartitionN"}, + ), nil +} + +// ValidateCreate gets run every time you call "pop.ValidateAndCreate" method. +// This method is not required and may be deleted. +func (t *Table) ValidateCreate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} + +// ValidateUpdate gets run every time you call "pop.ValidateAndUpdate" method. +// This method is not required and may be deleted. +func (t *Table) ValidateUpdate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} diff --git a/dax/models/worker.go b/dax/models/worker.go new file mode 100644 index 000000000..c2ddd25f7 --- /dev/null +++ b/dax/models/worker.go @@ -0,0 +1,60 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/gobuffalo/nulls" + "github.com/gobuffalo/pop/v6" + "github.com/gobuffalo/validate/v3" + "github.com/gobuffalo/validate/v3/validators" + "github.com/gofrs/uuid" +) + +// Worker is a node plus a role that gets assigned to a database and +// can be assigned jobs for that database. +type Worker struct { + ID uuid.UUID `json:"id" db:"id"` + Address dax.Address `json:"address" db:"address"` + Role dax.RoleType `json:"role" db:"role"` + DatabaseID nulls.String `json:"database_id" db:"database_id"` // this can be empty which means the worker is unassigned + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` + Jobs Jobs `json:"jobs" has_many:"jobs" order_by:"name asc"` +} + +// String is not required by pop and may be deleted +func (t *Worker) String() string { + jt, _ := json.MarshalIndent(t, " ", " ") //nolint:errchkjson + return string(jt) +} + +// Workers is not required by pop and may be deleted +type Workers []Worker + +// String is not required by pop and may be deleted +func (t Workers) String() string { + jt, _ := json.MarshalIndent(t, " ", " ") //nolint:errchkjson + return string(jt) +} + +// Validate gets run every time you call a "pop.Validate*" (pop.ValidateAndSave, pop.ValidateAndCreate, pop.ValidateAndUpdate) method. +// This method is not required and may be deleted. +func (t *Worker) Validate(tx *pop.Connection) (*validate.Errors, error) { + return validate.Validate( + &validators.StringIsPresent{Field: string(t.Address), Name: "Address"}, + ), nil +} + +// ValidateCreate gets run every time you call "pop.ValidateAndCreate" method. +// This method is not required and may be deleted. +func (t *Worker) ValidateCreate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} + +// ValidateUpdate gets run every time you call "pop.ValidateAndUpdate" method. +// This method is not required and may be deleted. +func (t *Worker) ValidateUpdate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} diff --git a/dax/queryer/http/handler.go b/dax/queryer/http/handler.go index fac16a6dd..b86671544 100644 --- a/dax/queryer/http/handler.go +++ b/dax/queryer/http/handler.go @@ -16,7 +16,17 @@ func Handler(q *queryer.Queryer) http.Handler { queryer: q, } + logRequestMiddleWare := func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.URL.Path, "/health") { + q.Logger().Debugf("serving %s, %v", r.Method, r.URL) + } + next.ServeHTTP(w, r) + }) + } + router := mux.NewRouter() + router.Use(logRequestMiddleWare) router.HandleFunc("/health", svr.getHealth).Methods("GET").Name("GetHealth") router.HandleFunc("/sql", svr.postSQL).Methods("POST").Name("PostSQL") router.HandleFunc("/databases/{databaseID}/sql", svr.postSQL).Methods("POST").Name("PostDatabaseSQL") diff --git a/dax/queryer/queryer.go b/dax/queryer/queryer.go index 205c10e9f..033db940a 100644 --- a/dax/queryer/queryer.go +++ b/dax/queryer/queryer.go @@ -57,6 +57,10 @@ func New(cfg Config) *Queryer { return q } +func (q *Queryer) Logger() logger.Logger { + return q.logger +} + // Orchestrator gets (or creates) an instance of qualifiedOrchestrator based on // the provided dax.QualifiedDatabaseID. func (q *Queryer) Orchestrator(qdbid dax.QualifiedDatabaseID) *qualifiedOrchestrator { @@ -265,7 +269,6 @@ func (q *Queryer) parseAndQueryPQL(ctx context.Context, qdbid dax.QualifiedDatab } table := sql[1:i] query := sql[i+1:] - fmt.Println("got table/query", table, query) return q.queryPQL(ctx, qdbid, dax.TableName(table), query) } diff --git a/dax/server/config.go b/dax/server/config.go index a909102b3..82a439f23 100644 --- a/dax/server/config.go +++ b/dax/server/config.go @@ -18,7 +18,7 @@ import ( const ( defaultBindPort = "8080" - defaultStorageMethod = "boltdb" + defaultStorageMethod = "sqldb" ) // Config represents the configuration for the command. @@ -71,6 +71,7 @@ func NewConfig() *Config { Config: controller.Config{ RegistrationBatchTimeout: time.Second * 3, StorageMethod: defaultStorageMethod, + SQLDB: controller.NewSQLDBConfig(), SnappingTurtleTimeout: time.Minute * 3, }, }, diff --git a/dax/server/test/managed.go b/dax/server/test/managed.go index efc3ca875..dbc298aee 100644 --- a/dax/server/test/managed.go +++ b/dax/server/test/managed.go @@ -15,12 +15,15 @@ import ( computersvc "github.com/featurebasedb/featurebase/v3/dax/computer/service" "github.com/featurebasedb/featurebase/v3/dax/controller" controllersvc "github.com/featurebasedb/featurebase/v3/dax/controller/service" + "github.com/featurebasedb/featurebase/v3/dax/controller/sqldb" "github.com/featurebasedb/featurebase/v3/dax/queryer" queryersvc "github.com/featurebasedb/featurebase/v3/dax/queryer/service" "github.com/featurebasedb/featurebase/v3/dax/server" "github.com/featurebasedb/featurebase/v3/errors" + "github.com/featurebasedb/featurebase/v3/logger" fbtest "github.com/featurebasedb/featurebase/v3/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // ManagedCommand represents a test wrapper for server.Command. @@ -201,7 +204,9 @@ func DefaultConfig() *server.Config { cfg := server.NewConfig() cfg.Verbose = true cfg.Controller.Run = true + cfg.Controller.Config.StorageMethod = "sqldb" cfg.Controller.Config.RegistrationBatchTimeout = 0 + cfg.Controller.Config.SQLDB = sqldb.GetTestConfig() cfg.Queryer.Run = true cfg.Computer.Run = true cfg.Computer.N = 1 @@ -226,6 +231,33 @@ func MustRunManagedCommand(tb testing.TB, opts ...server.CommandOption) *Managed mc := NewManagedCommand(tb, opts...) + var err error + testconf := sqldb.GetTestConfig() + fmt.Printf("testconf: %+v", *testconf) + trans, err := sqldb.Connect(testconf, logger.StderrLogger) + require.NoError(tb, err, "connecting") + + // The integration tests reuse the same database every time, but + // truncate all the tables *before* the tests run (rather than + // after). This has the advantage that if the tests fail partway + // through, you can inspect the state of the database for + // debugging purposes. + err = trans.TruncateAll() + if err != nil { + tb.Fatalf("truncating DB: %v", err) + } + + // The migrations contain an insert, but since we just truncated everything we need to redo that insert. + err = trans.RawQuery("INSERT INTO directive_versions (id, version, created_at, updated_at) VALUES (1, 1, '1970-01-01T00:00', '1970-01-01T00:00')").Exec() + if err != nil { + tb.Fatalf("reinserting directive_version record after truncation: %v", err) + } + + err = trans.Close() + if err != nil { + tb.Fatalf("Closing conn after truncating all tables: %v", err) + } + if err := mc.Start(); err != nil { tb.Fatalf("starting managed command: %v", err) } diff --git a/dax/storage/storage.go b/dax/storage/storage.go index d64c5e13e..cdc72fa32 100644 --- a/dax/storage/storage.go +++ b/dax/storage/storage.go @@ -161,21 +161,27 @@ func (mm *ResourceManager) RemoveAll() error { errList := make([]error, 0) for k, resource := range mm.shardResources { err := resource.Unlock() - if err != nil && !strings.Contains(err.Error(), "resource was not locked") { + if err != nil && + !strings.Contains(err.Error(), "resource was not locked") && + !strings.Contains(err.Error(), "no such file or directory") { errList = append(errList, err) } delete(mm.shardResources, k) } for k, resource := range mm.tableKeyResources { err := resource.Unlock() - if err != nil && !strings.Contains(err.Error(), "resource was not locked") { + if err != nil && + !strings.Contains(err.Error(), "resource was not locked") && + !strings.Contains(err.Error(), "no such file or directory") { errList = append(errList, err) } delete(mm.tableKeyResources, k) } for k, resource := range mm.fieldKeyResources { err := resource.Unlock() - if err != nil && !strings.Contains(err.Error(), "resource was not locked") { + if err != nil && + !strings.Contains(err.Error(), "resource was not locked") && + !strings.Contains(err.Error(), "no such file or directory") { errList = append(errList, err) } delete(mm.fieldKeyResources, k) diff --git a/dax/test/dax/dax_test.go b/dax/test/dax/dax_test.go index b9cfaec05..d00c347d2 100644 --- a/dax/test/dax/dax_test.go +++ b/dax/test/dax/dax_test.go @@ -17,6 +17,7 @@ import ( "github.com/featurebasedb/featurebase/v3/dax/controller" controllerclient "github.com/featurebasedb/featurebase/v3/dax/controller/client" "github.com/featurebasedb/featurebase/v3/dax/controller/schemar" + "github.com/featurebasedb/featurebase/v3/dax/controller/sqldb" queryerclient "github.com/featurebasedb/featurebase/v3/dax/queryer/client" "github.com/featurebasedb/featurebase/v3/dax/server" "github.com/featurebasedb/featurebase/v3/dax/server/test" @@ -61,7 +62,8 @@ func TestDAXIntegration(t *testing.T) { // Start ManagedCommand with an empty, new config. This results in a // ServiceManager running with no active services. They are added // later, throughout the test. - cfg := server.NewConfig() + cfg := server.NewConfig() // TODO: not sure why we're not getting test config here... had to add the SQLDB line below. + cfg.Controller.Config.SQLDB = sqldb.GetTestConfig() opt := server.OptCommandConfig(cfg) mc := test.MustRunManagedCommand(t, opt) defer mc.Close() @@ -246,18 +248,18 @@ func TestDAXIntegration(t *testing.T) { // ensure partitions are covered partitions0 := dax.PartitionNums{0, 2, 4, 6, 8, 10} partitions1 := dax.PartitionNums{1, 3, 5, 7, 9, 11} - allPartitions := append(partitions0, partitions1...) - sort.Sort(allPartitions) + allPartitionsExp := append(partitions0, partitions1...) - nodes, err := controllerClient.TranslateNodes(context.Background(), qtid, allPartitions...) + nodes, err := controllerClient.TranslateNodes(context.Background(), qtid, allPartitionsExp...) assert.NoError(t, err) if assert.Len(t, nodes, 2) { + + allParts := append(nodes[0].Partitions, nodes[1].Partitions...) + assert.ElementsMatch(t, allPartitionsExp, allParts) // computer0 (node0) assert.Equal(t, computers[computerKey0].Address(), nodes[0].Address) - assert.Equal(t, partitions0, nodes[0].Partitions) // computer1 (node1) assert.Equal(t, computers[computerKey1].Address(), nodes[1].Address) - assert.Equal(t, partitions1, nodes[1].Partitions) } // stop computer 0 (may need to sleep) @@ -273,7 +275,7 @@ func TestDAXIntegration(t *testing.T) { if assert.Len(t, nodes, 1) { // computer1 (node0) assert.Equal(t, computers[computerKey1].Address(), nodes[0].Address) - assert.Equal(t, allPartitions, nodes[0].Partitions) + assert.ElementsMatch(t, allPartitionsExp, nodes[0].Partitions) } }) @@ -600,10 +602,9 @@ func TestDAXIntegration(t *testing.T) { rootDir := mc.Config.Computer.Config.DataDir - // Ensure the index and writelogger directories are empty. + // Ensure the index and writelogger directories are *not* empty. assert.False(t, dirIsEmpty(t, rootDir+"/computer0")) assert.False(t, dirIsEmpty(t, rootDir+"/computer0/indexes")) - assert.False(t, dirIsEmpty(t, rootDir+"/controller")) assert.False(t, dirIsEmpty(t, rootDir+"/wl")) resp := runSQL(t, svcmgr.Queryer.Address(), testconfigs[0].qdbid, "drop table keyed") @@ -612,7 +613,6 @@ func TestDAXIntegration(t *testing.T) { // Ensure the index and writelogger directories are empty. assert.False(t, dirIsEmpty(t, rootDir+"/computer0")) assert.True(t, dirIsEmpty(t, rootDir+"/computer0/indexes")) - assert.False(t, dirIsEmpty(t, rootDir+"/controller")) assert.True(t, dirIsEmpty(t, rootDir+"/wl")) }) @@ -636,7 +636,7 @@ func TestDAXIntegration(t *testing.T) { computerKey0 := dax.ServiceKey(dax.ServicePrefixComputer + "0") computerKey1 := dax.ServiceKey(dax.ServicePrefixComputer + "1") computerKey2 := dax.ServiceKey(dax.ServicePrefixComputer + "2") - // computerKey3 := dax.ServiceKey(dax.ServicePrefixComputer + "3") + computerKey3 := dax.ServiceKey(dax.ServicePrefixComputer + "3") // Ingest and query some data. runTableTests(t, @@ -648,20 +648,13 @@ func TestDAXIntegration(t *testing.T) { assert.NoError(t, err) // ensure partitions are covered - partitions0 := dax.PartitionNums{0, 2, 4, 6, 8, 10} - partitions1 := dax.PartitionNums{1, 3, 5, 7, 9, 11} - allPartitions := append(partitions0, partitions1...) - sort.Sort(allPartitions) + allPartitions := dax.PartitionNums{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11} nodes, err := controllerClient.TranslateNodes(context.Background(), qtid, allPartitions...) assert.NoError(t, err) if assert.Len(t, nodes, 2) { - // computer0 (node0) - assert.Equal(t, computers[computerKey0].Address(), nodes[0].Address) - assert.Equal(t, partitions0, nodes[0].Partitions) - // computer1 (node1) - assert.Equal(t, computers[computerKey1].Address(), nodes[1].Address) - assert.Equal(t, partitions1, nodes[1].Partitions) + gotPartitions := append(nodes[0].Partitions, nodes[1].Partitions...) + assert.ElementsMatch(t, allPartitions, gotPartitions) } // Change DatabaseOptions.WorkersMin to 3. @@ -681,25 +674,16 @@ func TestDAXIntegration(t *testing.T) { ) }) - // ensure partitions are still covered - partitions0 = dax.PartitionNums{0, 10} - partitions1 = dax.PartitionNums{1, 11} - partitions2 := dax.PartitionNums{2, 3, 4, 5, 6, 7, 8, 9} - allPartitions = append(append(partitions0, partitions1...), partitions2...) - sort.Sort(allPartitions) - nodes, err = controllerClient.TranslateNodes(context.Background(), qtid, allPartitions...) assert.NoError(t, err) if assert.Len(t, nodes, 3) { + gotPartitions := append(nodes[0].Partitions, append(nodes[1].Partitions, nodes[2].Partitions...)...) // computer0 (node0) - assert.Equal(t, computers[computerKey0].Address(), nodes[0].Address) - assert.Equal(t, partitions0, nodes[0].Partitions) - // computer1 (node1) - assert.Equal(t, computers[computerKey1].Address(), nodes[1].Address) - assert.Equal(t, partitions1, nodes[1].Partitions) - // computer2 (node2) - assert.Equal(t, computers[computerKey2].Address(), nodes[2].Address) - assert.Equal(t, partitions2, nodes[2].Partitions) + assert.ElementsMatch(t, allPartitions, gotPartitions) + expKeys := []dax.Address{computers[computerKey0].Address(), computers[computerKey1].Address(), + computers[computerKey2].Address(), computers[computerKey3].Address()} + gotKeys := []dax.Address{nodes[0].Address, nodes[1].Address, nodes[2].Address} + assert.Subset(t, expKeys, gotKeys) } }) @@ -745,73 +729,53 @@ func TestDAXIntegration(t *testing.T) { t.Run("CreateDatabase", func(t *testing.T) { err := client.CreateDatabase(ctx, nil) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, schemar.ErrCodeDatabaseNameInvalid)) - } + assertCode(t, err, schemar.ErrCodeDatabaseNameInvalid) }) t.Run("DropDatabase", func(t *testing.T) { err := client.DropDatabase(ctx, qdbid) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrDatabaseIDDoesNotExist)) - } + assertCode(t, err, dax.ErrDatabaseIDDoesNotExist) }) t.Run("DatabaseByName", func(t *testing.T) { _, err := client.DatabaseByName(ctx, "", "") - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrDatabaseNameDoesNotExist)) - } + assertCode(t, err, dax.ErrDatabaseNameDoesNotExist) }) t.Run("DatabaseByID", func(t *testing.T) { _, err := client.DatabaseByID(ctx, qdbid) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrDatabaseIDDoesNotExist)) - } + assertCode(t, err, dax.ErrDatabaseIDDoesNotExist) }) t.Run("SetDatabaseOption", func(t *testing.T) { - err := client.SetDatabaseOption(ctx, qdbid, "", "") - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrDatabaseIDDoesNotExist)) - } + err := client.SetDatabaseOption(ctx, qdbid, dax.DatabaseOptionWorkersMin, "9") + assertCode(t, err, dax.ErrDatabaseIDDoesNotExist) }) t.Run("Databases", func(t *testing.T) { _, err := client.Databases(ctx, "", dbID) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrOrganizationIDDoesNotExist)) - } + assertCode(t, err, dax.ErrOrganizationIDDoesNotExist) }) t.Run("CreateTable", func(t *testing.T) { err := client.CreateTable(ctx, &qtbl) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, schemar.ErrCodeTableNameInvalid)) - } + assertCode(t, err, schemar.ErrCodeTableNameInvalid) }) t.Run("DropTable", func(t *testing.T) { err := client.DropTable(ctx, qtid) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist)) - } + assertCode(t, err, dax.ErrTableIDDoesNotExist) }) t.Run("TableByName", func(t *testing.T) { req := dax.QualifiedTableID{} _, err := client.TableByName(ctx, qdbid, req.Name) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrTableNameDoesNotExist)) - } + assertCode(t, err, dax.ErrTableNameDoesNotExist) }) t.Run("TableByID", func(t *testing.T) { _, err := client.TableByID(ctx, qtid) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist)) - } + assertCode(t, err, dax.ErrTableIDDoesNotExist) }) //Todo: make it so "Tables" doesn't return all tables if error is present @@ -825,16 +789,12 @@ func TestDAXIntegration(t *testing.T) { t.Run("CreateField", func(t *testing.T) { err := client.CreateField(ctx, qtid, tbfld) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, schemar.ErrCodeFieldNameInvalid)) - } + assertCode(t, err, schemar.ErrCodeFieldNameInvalid) }) t.Run("DropField", func(t *testing.T) { err := client.DropField(ctx, qtid, tbfld.Name) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist)) - } + assertCode(t, err, dax.ErrFieldDoesNotExist) }) }) }) @@ -1164,3 +1124,9 @@ func sortStringKeys(in [][]interface{}) { } } } + +func assertCode(t *testing.T, err error, code errors.Code) { + if !errors.Is(err, code) { + t.Errorf("Error '%v' does not have code %s.", err, code) + } +} diff --git a/errors/errors.go b/errors/errors.go index dabc52d20..1269cd2a0 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -74,6 +74,9 @@ type codedError struct { } func (ce codedError) Error() string { + if ce.Wrapped != "" { + return ce.Wrapped + } return ce.Message } diff --git a/go.mod b/go.mod index 6a9e6a90e..c218b2e7d 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/glycerine/vprint v0.0.0-20200730000117-76cea49a68ea github.com/go-avro/avro v0.0.0-20171219232920-444163702c11 github.com/go-openapi/strfmt v0.21.2 // indirect - github.com/go-sql-driver/mysql v1.6.0 + github.com/go-sql-driver/mysql v1.7.0 github.com/go-test/deep v1.0.7 github.com/gogo/protobuf v1.3.2 github.com/golang-jwt/jwt v3.2.2+incompatible @@ -38,7 +38,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.1 github.com/improbable-eng/grpc-web v0.15.0 github.com/jedib0t/go-pretty v4.3.0+incompatible - github.com/lib/pq v1.10.5 + github.com/lib/pq v1.10.7 github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b github.com/opentracing/opentracing-go v1.2.0 github.com/pelletier/go-toml v1.9.5 @@ -53,10 +53,10 @@ require ( github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76 github.com/segmentio/kafka-go v0.4.29 github.com/shirou/gopsutil/v3 v3.22.5 - github.com/spf13/cobra v1.2.1 + github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 github.com/uber/jaeger-client-go v2.25.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/zeebo/blake3 v0.2.3 @@ -84,6 +84,10 @@ require ( github.com/PaesslerAG/jsonpath v0.1.1 github.com/apache/arrow/go/v10 v10.0.0-20221021053532-2f627c213fc3 github.com/benhoyt/goawk v1.21.0 + github.com/gobuffalo/nulls v0.4.2 + github.com/gobuffalo/pop/v6 v6.1.1 + github.com/gobuffalo/validate/v3 v3.3.3 + github.com/gofrs/uuid v4.3.1+incompatible github.com/gomem/gomem v0.1.0 github.com/google/uuid v1.3.0 github.com/jaffee/commandeer v0.6.0 @@ -96,10 +100,42 @@ require ( require ( github.com/DataDog/datadog-agent/pkg/obfuscate v0.0.0-20211129110424-6491aa3bf583 // indirect github.com/DataDog/sketches-go v1.0.0 // indirect + github.com/Masterminds/semver/v3 v3.1.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect + github.com/fatih/color v1.13.0 // indirect + github.com/fatih/structs v1.1.0 // indirect + github.com/gobuffalo/envy v1.10.2 // indirect + github.com/gobuffalo/fizz v1.14.4 // indirect + github.com/gobuffalo/flect v1.0.0 // indirect + github.com/gobuffalo/github_flavored_markdown v1.1.3 // indirect + github.com/gobuffalo/helpers v0.6.7 // indirect + github.com/gobuffalo/plush/v4 v4.1.18 // indirect + github.com/gobuffalo/tags/v3 v3.1.4 // indirect + github.com/gorilla/css v1.0.0 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.13.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.1 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.12.0 // indirect + github.com/jackc/pgx/v4 v4.17.2 // indirect + github.com/jmoiron/sqlx v1.3.5 // indirect + github.com/joho/godotenv v1.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/luna-duclos/instrumentedsql v1.1.3 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.11 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-sqlite3 v1.14.16 // indirect + github.com/microcosm-cc/bluemonday v1.0.20 // indirect github.com/philhofer/fwd v1.1.1 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/sergi/go-diff v1.2.0 // indirect + github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d // indirect + github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e // indirect github.com/tinylib/msgp v1.1.2 // indirect ) @@ -138,7 +174,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -161,12 +197,12 @@ require ( github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - github.com/sirupsen/logrus v1.7.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/stretchr/objx v0.4.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect diff --git a/go.sum b/go.sum index 98606216a..e90e72013 100644 --- a/go.sum +++ b/go.sum @@ -70,6 +70,7 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXY github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= @@ -135,6 +136,8 @@ github.com/aws/aws-sdk-go-v2/service/sqs v1.0.0/go.mod h1:w5BclCU8ptTbagzXS/fHBr github.com/aws/aws-sdk-go-v2/service/sts v1.0.0/go.mod h1:5f+cELGATgill5Pu3/vK3Ebuigstc+qYEHW5MvGWZO4= github.com/aws/smithy-go v1.0.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= github.com/aws/smithy-go v1.11.0/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/benbjohnson/immutable v0.4.0 h1:CTqXbEerYso8YzVPxmWxh2gnoRQbbB9X1quUC8+vGZA= @@ -187,6 +190,7 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= @@ -215,6 +219,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -260,8 +265,10 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/fgprof v0.9.2 h1:tAMHtWMyl6E0BimjVbFt7fieU6FpjttsZN7j0wT5blc= github.com/felixge/fgprof v0.9.2/go.mod h1:+VNi+ZXtHIQ6wIw6bUT8nXQRefQflWECoFyRealT5sg= @@ -338,8 +345,9 @@ github.com/go-redis/redis/v7 v7.1.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRf github.com/go-redis/redis/v8 v8.0.0/go.mod h1:isLoQT/NFSP7V67lyvM9GmdvLdyZ7pEhsXvvyQtnQTo= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -348,29 +356,55 @@ github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3a github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/attrs v1.0.3/go.mod h1:KvDJCE0avbufqS0Bw3UV7RQynESY0jjod+572ctX4t8= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.10.2 h1:EIi03p9c3yeuRCFPOKcSfajzkLb3hrRjEpHGI8I2Wo4= +github.com/gobuffalo/envy v1.10.2/go.mod h1:qGAGwdvDsaEtPhfBzb3o0SfDea8ByGn9j8bKmVft9z8= +github.com/gobuffalo/fizz v1.14.4 h1:8uume7joF6niTNWN582IQ2jhGTUoa9g1fiV/tIoGdBs= +github.com/gobuffalo/fizz v1.14.4/go.mod h1:9/2fGNXNeIFOXEEgTPJwiK63e44RjG+Nc4hfMm1ArGM= github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.3.0/go.mod h1:5pf3aGnsvqvCj50AVni7mJJF8ICxGZ8HomberC3pXLE= +github.com/gobuffalo/flect v1.0.0 h1:eBFmskjXZgAOagiTXJH25Nt5sdFwNRcb8DKZsIsAUQI= +github.com/gobuffalo/flect v1.0.0/go.mod h1:l9V6xSb4BlXwsxEMj3FVEub2nkdQjWhPvD8XTTlHPQc= github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/genny/v2 v2.1.0/go.mod h1:4yoTNk4bYuP3BMM6uQKYPvtP6WsXFGm2w2EFYZdRls8= github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/github_flavored_markdown v1.1.3 h1:rSMPtx9ePkFB22vJ+dH+m/EUBS8doQ3S8LeEXcdwZHk= +github.com/gobuffalo/github_flavored_markdown v1.1.3/go.mod h1:IzgO5xS6hqkDmUh91BW/+Qxo/qYnvfzoz3A7uLkg77I= github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/helpers v0.6.7 h1:C9CedoRSfgWg2ZoIkVXgjI5kgmSpL34Z3qdnzpfNVd8= +github.com/gobuffalo/helpers v0.6.7/go.mod h1:j0u1iC1VqlCaJEEVkZN8Ia3TEzfj/zoXANqyJExTMTA= github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/logger v1.0.7/go.mod h1:u40u6Bq3VVvaMcy5sRBclD8SXhBYPS0Qk95ubt+1xJM= github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/nulls v0.4.2 h1:GAqBR29R3oPY+WCC7JL9KKk9erchaNuV6unsOSZGQkw= +github.com/gobuffalo/nulls v0.4.2/go.mod h1:EElw2zmBYafU2R9W4Ii1ByIj177wA/pc0JdjtD0EsH8= github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v1.0.2/go.mod h1:sUc61tDqGMXON80zpKGp92lDb86Km28jfvX7IAyxFT8= github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/plush/v4 v4.1.16/go.mod h1:6t7swVsarJ8qSLw1qyAH/KbrcSTwdun2ASEQkOznakg= +github.com/gobuffalo/plush/v4 v4.1.18 h1:bnPjdMTEUQHqj9TNX2Ck3mxEXYZa+0nrFMNM07kpX9g= +github.com/gobuffalo/plush/v4 v4.1.18/go.mod h1:xi2tJIhFI4UdzIL8sxZtzGYOd2xbBpcFbLZlIPGGZhU= +github.com/gobuffalo/pop/v6 v6.1.1 h1:eUDBaZcb0gYrmFnKwpuTEUA7t5ZHqNfvS4POqJYXDZY= +github.com/gobuffalo/pop/v6 v6.1.1/go.mod h1:1n7jAmI1i7fxuXPZjZb0VBPQDbksRtCoFnrDV5IsvaI= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gobuffalo/tags/v3 v3.1.4 h1:X/ydLLPhgXV4h04Hp2xlbI2oc5MDaa7eub6zw8oHjsM= +github.com/gobuffalo/tags/v3 v3.1.4/go.mod h1:ArRNo3ErlHO8BtdA0REaZxijuWnWzF6PUXngmMXd2I0= +github.com/gobuffalo/validate/v3 v3.3.3 h1:o7wkIGSvZBYBd6ChQoLxkz2y1pfmhbI4jNJYh6PuNJ4= +github.com/gobuffalo/validate/v3 v3.3.3/go.mod h1:YC7FsbJ/9hW/VjQdmXPvFqvRis4vrRYFxr69WiNZw6g= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 h1:YyrUZvJaU8Q0QsoVo+xLFBgWDTam29PKea6GYmwvSiQ= github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= @@ -387,6 +421,9 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gofiber/fiber/v2 v2.11.0/go.mod h1:oZTLWqYnqpMMuF922SjGbsYZsdpE1MCfh416HNdweIM= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI= +github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -500,6 +537,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0= github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/handlers v1.3.0 h1:tsg9qP3mjt1h4Roxp+M1paRjrVBfPSOpBuVclh6YluI= github.com/gorilla/handlers v1.3.0/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.5.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -594,12 +633,14 @@ github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ= github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/invopop/jsonschema v0.4.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= @@ -612,10 +653,15 @@ github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfG github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= github.com/jackc/pgconn v1.10.1/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= +github.com/jackc/pgconn v1.13.0/go.mod h1:AnowpAqO4CMIIJNZl2VJp+KrkAZciAkhEl0W0JIobpI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= @@ -627,7 +673,10 @@ github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y= +github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= @@ -638,6 +687,8 @@ github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/V github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= github.com/jackc/pgtype v1.9.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= +github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= @@ -647,12 +698,15 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9s github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.14.0/go.mod h1:jT3ibf/A0ZVCp89rtCIN0zCJxcE74ypROmHEZYsG/j8= +github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E= +github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jaffee/commandeer v0.6.0 h1:YI44XLWcJN21euhh32sZW8vM/tljPYxhsXIfEPkQKcs= github.com/jaffee/commandeer v0.6.0/go.mod h1:kCwfuSvZ2T0NVEr3LDSo6fDUgi0xSBnAVDdkOKTtpLQ= github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= @@ -672,7 +726,11 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= +github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= @@ -699,6 +757,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -742,8 +802,8 @@ github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ= -github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM= @@ -753,6 +813,8 @@ github.com/linkedin/goavro/v2 v2.11.1 h1:4cuAtbDfqkKnBXp9E+tRkIJGa6W6iAjwonwt8O1 github.com/linkedin/goavro/v2 v2.11.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/luna-duclos/instrumentedsql v1.1.3 h1:t7mvC0z1jUt5A0UQ6I/0H31ryymuQRnJcWCiqV3lSAA= +github.com/luna-duclos/instrumentedsql v1.1.3/go.mod h1:9J1njvFds+zN7y85EDhN9XNQLANWwZt2ULeIC8yMNYs= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -772,6 +834,7 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -785,12 +848,19 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y= +github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= @@ -974,6 +1044,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= @@ -981,6 +1052,7 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= @@ -992,10 +1064,13 @@ github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76/go.mod h1:dA0hQrY github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/kafka-go v0.4.29 h1:4ujULpikzHG0HqKhjumDghFjy/0RRCSl/7lbriwQAH0= github.com/segmentio/kafka-go v0.4.29/go.mod h1:m1lXeqJtIFYZayv0shM/tjrAFljvWLTprxBHd+3PnaU= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil/v3 v3.22.5 h1:atX36I/IXgFiB81687vSiBI5zrMsxcIBkP9cQMJQoJA= github.com/shirou/gopsutil/v3 v3.22.5/go.mod h1:so9G9VzeHt/hsd0YwqprnjHnfARAUktauykSbr+y2gA= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -1003,8 +1078,9 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= @@ -1014,6 +1090,10 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d h1:yKm7XZV6j9Ev6lojP2XaIshpT4ymkqhMeSghO5Ps00E= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e h1:qpG93cPwA5f7s/ZPBJnGOYQNK/vKsaDaseuKT5Asee8= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -1025,8 +1105,8 @@ github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -1045,8 +1125,9 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1056,8 +1137,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= @@ -1135,6 +1217,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= @@ -1244,6 +1327,7 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1291,6 +1375,7 @@ golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hM golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1354,6 +1439,9 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1384,6 +1472,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1477,11 +1567,17 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1574,6 +1670,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=