From a1fc6d04a1fa5afda00ee6903e7c00a3fd4b996a Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula <85502298+pokeeffe-molecula@users.noreply.github.com> Date: Wed, 11 Jan 2023 20:37:00 -0600 Subject: [PATCH 01/15] introduce performance counters and system table fanout, plus refactor metrics (#2363) * performance counters * first cut of perf counters and system table fanout and a wire protocol * significantly refactored prometheus support; removed statsd and exprvar * removed node_id * put dax subquery test back * Change Translator.TranslateFieldIDs method to take a dax.TableKeyer There are a bunch of other calls to the Translator interface methods with currently take an `index string`, and those need to be converted to dax.TableKeyer as well. But I need to review each call, because in at least one place I noticed one being called with `result.Index` instead of with the qtbl available. And I don't yet know how those could be different. Co-authored-by: Travis Turner (cherry picked from commit 7f6ea0e6e5eef76a6bea308743cb4965b4c1ab3c) --- Dockerfile-clustertests | 4 +- Dockerfile-clustertests-client | 6 +- api.go | 46 +- batch/Dockerfile-test | 4 +- batch/batch.go | 8 +- batch/metrics.go | 25 - cache.go | 50 +- client/client.go | 15 - client/importer.go | 4 - dax/README.md | 12 +- dax/queryer/orchestrator.go | 101 ++- dax/queryer/queryer.go | 1 - dax/queryer/translator.go | 10 +- dax/test/dax/dax_test.go | 1 - executor.go | 104 ++- field.go | 3 - fragment.go | 18 +- go.mod | 2 +- holder.go | 14 - http_handler.go | 108 ++- idk/Makefile | 2 +- idk/ingest.go | 69 +- idk/ingest_test.go | 2 +- idk/mds/importer.go | 2 - idk/metrics.go | 44 ++ importer.go | 2 - index.go | 6 +- lattice/README.md | 2 +- metrics.go | 996 ++++++++++++++++++++++++++++ performancecounters.go | 203 ++++++ prometheus/prometheus.go | 308 --------- prometheus/prometheus_test.go | 48 +- row.go | 4 +- server.go | 35 +- server/grpc.go | 42 +- server/handler_test.go | 9 - server/server.go | 25 - sql3/interfaces.go | 6 + sql3/planner/compilecreatetable.go | 8 +- sql3/planner/compileselect.go | 9 +- sql3/planner/executionplanner.go | 232 +++++++ sql3/planner/opbulkinsert.go | 12 + sql3/planner/opfanout.go | 114 ++++ sql3/planner/opinsert.go | 3 + sql3/planner/opsystemtable.go | 98 ++- sql3/planner/types/operator.go | 5 +- sql3/planner/wireprotocol.go | 211 ++++++ sql3/sql_complex_test.go | 140 ++-- stats/stats.go | 274 -------- stats/stats_test.go | 258 ------- statsd/statsd.go | 152 ----- statsd/statsd_test.go | 50 -- view.go | 3 - wireprotocol/wireprimitives.go | 563 ++++++++++++++++ wireprotocol/wireprimitives_test.go | 144 ++++ 55 files changed, 3083 insertions(+), 1534 deletions(-) create mode 100644 performancecounters.go delete mode 100644 prometheus/prometheus.go create mode 100644 sql3/planner/opfanout.go create mode 100644 sql3/planner/wireprotocol.go delete mode 100644 stats/stats.go delete mode 100644 stats/stats_test.go delete mode 100644 statsd/statsd.go delete mode 100644 statsd/statsd_test.go create mode 100644 wireprotocol/wireprimitives.go create mode 100644 wireprotocol/wireprimitives_test.go diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests index 11160bbef..086bea738 100644 --- a/Dockerfile-clustertests +++ b/Dockerfile-clustertests @@ -20,12 +20,12 @@ RUN apt install -y docker.io ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose RUN chmod +x /usr/local/bin/docker-compose -WORKDIR /go/src/github.com/molecula/featurebase/cmd/featurebase +WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase # generate an instrumented binary to allow for calculating code coverage for clustertests # the entrypoint for the binary is TestRunMain, which is wrapper for main RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase -RUN cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase +RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase COPY NOTICE /NOTICE diff --git a/Dockerfile-clustertests-client b/Dockerfile-clustertests-client index b3290b032..9983cf8fe 100644 --- a/Dockerfile-clustertests-client +++ b/Dockerfile-clustertests-client @@ -19,10 +19,10 @@ RUN apt install -y docker.io ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose RUN chmod +x /usr/local/bin/docker-compose -WORKDIR /go/src/github.com/molecula/featurebase/cmd/featurebase +WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase -RUN cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase +RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase COPY NOTICE /NOTICE @@ -32,6 +32,6 @@ COPY ./internal/clustertests /go/src/github.com/featurebasedb/featurebase/intern EXPOSE 10101 VOLUME /data -WORKDIR /go/src/github.com/molecula/featurebase +WORKDIR /go/src/github.com/featurebasedb/featurebase CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"] diff --git a/api.go b/api.go index be059e4f0..ae61eb99a 100644 --- a/api.go +++ b/api.go @@ -278,7 +278,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index return nil, errors.Wrap(err, "creating index") } - api.holder.Stats.Count(MetricCreateIndex, 1, 1.0) + CounterCreateIndex.Inc() return index, nil } @@ -319,7 +319,7 @@ func (api *API) DeleteDataframe(ctx context.Context, indexName string) error { api.server.logger.Errorf("problem sending DeleteIndex message: %s", err) return errors.Wrap(err, "sending DeleteIndex message") } - api.holder.Stats.Count(MetricDeleteDataframe, 1, 1.0) + CounterDeleteDataframe.Inc() return nil } @@ -354,7 +354,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { return errors.Wrap(err, "deleting id allocation for index") } } - api.holder.Stats.Count(MetricDeleteIndex, 1, 1.0) + CounterDeleteIndex.Inc() return nil } @@ -406,7 +406,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str return nil, errors.Wrap(err, "sending CreateField message") } - api.holder.Stats.CountWithCustomTags(MetricCreateField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + CounterCreateField.With(prometheus.Labels{"index": indexName}) return field, nil } @@ -750,7 +750,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str api.server.logger.Errorf("problem sending DeleteField message: %s", err) return errors.Wrap(err, "sending DeleteField message") } - api.holder.Stats.CountWithCustomTags(MetricDeleteField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + CounterDeleteField.With(prometheus.Labels{"index": indexName}) return nil } @@ -782,7 +782,7 @@ func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName str api.server.logger.Errorf("problem sending DeleteAvailableShard message: %s", err) return errors.Wrap(err, "sending DeleteAvailableShard message") } - api.holder.Stats.CountWithCustomTags(MetricDeleteAvailableShard, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + CounterDeleteAvailableShard.With(prometheus.Labels{"index": indexName}).Inc() return nil } @@ -2096,15 +2096,6 @@ func (api *API) AvailableShards(ctx context.Context, indexName string) (*roaring return index.AvailableShards(false), nil } -// StatsWithTags returns an instance of whatever implementation of StatsClient -// pilosa is using with the given tags. -func (api *API) StatsWithTags(tags []string) stats.StatsClient { - if api.holder == nil || api.cluster == nil { - return nil - } - return api.holder.Stats.WithTags(tags...) -} - // LongQueryTime returns the configured threshold for logging/statting // long running queries. func (api *API) LongQueryTime() time.Duration { @@ -2391,19 +2382,19 @@ func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Du switch err { case nil: if exclusive { - api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0) + CounterExclusiveTransactionRequest.Inc() } else { - api.holder.Stats.Count(MetricTransactionStart, 1, 1.0) + CounterTransactionStart.Inc() } case ErrTransactionExclusive: if exclusive { - api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0) + CounterExclusiveTransactionBlocked.Inc() } else { - api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0) + CounterTransactionBlocked.Inc() } } if exclusive && t != nil && t.Active { - api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0) + CounterExclusiveTransactionActive.Inc() } return t, err } @@ -2415,9 +2406,9 @@ func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) ( t, err := api.server.FinishTransaction(ctx, id, remote) if err == nil { if t.Exclusive { - api.holder.Stats.Count(MetricExclusiveTransactionEnd, 1, 1.0) + CounterExclusiveTransactionEnd.Inc() } else { - api.holder.Stats.Count(MetricTransactionEnd, 1, 1.0) + CounterTransactionEnd.Inc() } } return t, err @@ -2437,7 +2428,7 @@ func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Tr t, err := api.server.GetTransaction(ctx, id, remote) if err == nil { if t.Exclusive && t.Active { - api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0) + CounterExclusiveTransactionActive.Inc() } } return t, err @@ -3046,6 +3037,10 @@ func (api *API) CompilePlan(ctx context.Context, q string) (planner_types.PlanOp return api.server.CompileExecutionPlan(ctx, q) } +func (api *API) RehydratePlanOperator(ctx context.Context, reader io.Reader) (planner_types.PlanOperator, error) { + return api.server.RehydratePlanOperator(ctx, reader) +} + func (api *API) RBFDebugInfo() map[string]*rbf.DebugInfo { infos := make(map[string]*rbf.DebugInfo) @@ -3343,6 +3338,7 @@ type SystemAPI interface { ClusterState() string DataDir() string + NodeID() string ClusterNodes() []ClusterNode } @@ -3414,6 +3410,10 @@ func (fsapi *FeatureBaseSystemAPI) DataDir() string { return fsapi.server.dataDir } +func (fsapi *FeatureBaseSystemAPI) NodeID() string { + return fsapi.cluster.Node.ID +} + func (fsapi *FeatureBaseSystemAPI) ClusterNodes() []ClusterNode { result := make([]ClusterNode, 0) diff --git a/batch/Dockerfile-test b/batch/Dockerfile-test index dd6ca6962..a938392e5 100644 --- a/batch/Dockerfile-test +++ b/batch/Dockerfile-test @@ -2,10 +2,10 @@ ARG GO_VERSION=1.19 FROM golang:${GO_VERSION} -WORKDIR /go/src/github.com/molecula/featurebase/ +WORKDIR /go/src/github.com/featurebasedb/featurebase/ COPY . . -WORKDIR /go/src/github.com/molecula/featurebase/batch/ +WORKDIR /go/src/github.com/featurebasedb/featurebase/batch/ CMD ["go","test","-v","-mod=vendor","-tags=odbc,dynamic","./..."] diff --git a/batch/batch.go b/batch/batch.go index ca1d3ef97..4b3a90bcc 100644 --- a/batch/batch.go +++ b/batch/batch.go @@ -755,7 +755,7 @@ func (b *Batch) Import() error { }() } defer func() { - b.importer.StatsTiming(MetricBatchImportDurationSeconds, time.Since(start), 1.0) + featurebase.SummaryBatchImportDurationSeconds.Observe(time.Since(start).Seconds()) }() size := len(b.ids) @@ -828,7 +828,7 @@ func (b *Batch) Flush() error { if err != nil { b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl) } - b.importer.StatsTiming(MetricBatchFlushDurationSeconds, time.Since(start), 1.0) + featurebase.SummaryBatchFlushDurationSeconds.Observe(time.Since(start).Seconds()) }() importStart := time.Now() @@ -1188,7 +1188,7 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error { } } - b.importer.StatsTiming(MetricBatchShardImportBuildRequestsSeconds, time.Since(start), 1.0) + featurebase.SummaryBatchShardImportBuildRequestsSeconds.Observe(time.Since(start).Seconds()) start = time.Now() eg := egpool.Group{PoolSize: 20} for shard, request := range requests { @@ -1200,7 +1200,7 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error { } err := eg.Wait() dur := time.Since(start) - b.importer.StatsTiming(MetricBatchShardImportDurationSeconds, dur, 1.0) + featurebase.SummaryBatchImportDurationSeconds.Observe(dur.Seconds()) b.log.Printf("import shard took: %v\n", dur) return errors.Wrap(err, "doing shard-transactional imports") } diff --git a/batch/metrics.go b/batch/metrics.go index 591c7765c..873206fe2 100644 --- a/batch/metrics.go +++ b/batch/metrics.go @@ -1,28 +1,3 @@ // Copyright 2022 Molecula Corp. (DBA FeatureBase). // SPDX-License-Identifier: Apache-2.0 package batch - -const ( - // MetricBatchImportDurationSeconds records the full time of the - // RecordBatch.Import call. This includes starting and finishing a - // transaction, doing key translation, building fragments locally, - // importing all data, and resetting internal structures. - MetricBatchImportDurationSeconds = "batch_import_duration_seconds" - - // MetricBatchFlushDurationSeconds records the full time for - // RecordBatch.Flush (if splitBatchMode is in use). This includes - // starting and finishing a transaction, importing all data, and - // resetting internal structures. - MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds" - - // MetricBatchShardImportBuildRequestsSeconds is the time it takes - // after making fragments to build the shard-transactional request - // objects (but not actually import them or do any network activity). - MetricBatchShardImportBuildRequestsSeconds = "batch_shard_import_build_requests_seconds" - - // MetricBatchShardImportDurationSeconds is the time it takes to - // import all data for all shards in the batch using the - // shard-transactional endpoint. This does not include the time it - // takes to build the requests locally. - MetricBatchShardImportDurationSeconds = "batch_shard_import_duration_seconds" -) diff --git a/cache.go b/cache.go index 90cf42d3e..4577d69c9 100644 --- a/cache.go +++ b/cache.go @@ -41,9 +41,6 @@ type cache interface { // Returns an ordered list of the top ranked bitmaps. Top() []bitmapPair - // SetStats defines the stats client used in the cache. - SetStats(s stats.StatsClient) - // Clear removes everything from the cache. If possible it should leave allocated structures in place to be reused. Clear() } @@ -52,7 +49,6 @@ type cache interface { type lruCache struct { cache *lru.Cache counts map[uint64]uint64 - stats stats.StatsClient // maxEntries is saved to support Clear which recreates the cache. maxEntries uint32 } @@ -62,7 +58,6 @@ func newLRUCache(maxEntries uint32) *lruCache { c := &lruCache{ cache: lru.New(int(maxEntries)), counts: make(map[uint64]uint64), - stats: stats.NopStatsClient, maxEntries: maxEntries, } c.cache.OnEvicted = c.onEvicted @@ -120,11 +115,6 @@ func (c *lruCache) Top() []bitmapPair { return a } -// SetStats defines the stats client used in the cache. -func (c *lruCache) SetStats(s stats.StatsClient) { - c.stats = s -} - func (c *lruCache) Clear() { for k := range c.counts { delete(c.counts, k) @@ -158,8 +148,6 @@ type rankCache struct { // thresholdValue is the value of the last item in the cache thresholdValue uint64 - - stats stats.StatsClient } // NewRankCache returns a new instance of RankCache. @@ -168,7 +156,6 @@ func NewRankCache(maxEntries uint32) *rankCache { maxEntries: maxEntries, thresholdBuffer: int(thresholdFactor * float64(maxEntries)), entries: make(map[uint64]uint64), - stats: stats.NopStatsClient, } } @@ -229,7 +216,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) { // as this can take up an upbounded amount of memory. This is especially // true when restoring shards as all rows will be added. if len(c.entries) > int(2*c.maxEntries) { - c.stats.Count(MetricRecalculateCache, 1, 1.0) + CounterRecalculateCache.Inc() c.recalculate() } } @@ -274,7 +261,7 @@ func (c *rankCache) Invalidate() { func (c *rankCache) Recalculate() { c.mu.Lock() defer c.mu.Unlock() - c.stats.Count(MetricRecalculateCache, 1, 1.0) + CounterRecalculateCache.Inc() c.recalculate() } @@ -286,12 +273,12 @@ func (c *rankCache) invalidate() { // This is somewhat necessary for now since recalculation is not cheap. // The cache will remain flagged as dirty and will be recalculated if Top is called. // This may cause unexpected memory growth, so record it in metrics for debugging purposes. - c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0) + CounterInvalidateCacheSkipped.Inc() // Ensure that we're marked as dirty even if we weren't otherwise. c.dirty = true return } - c.stats.Count(MetricInvalidateCache, 1, 1.0) + CounterInvalidateCache.Inc() c.recalculate() } @@ -317,7 +304,7 @@ func (c *rankCache) recalculate() { // Store the count of the item at the threshold index. length := len(c.rankings) - c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0) + GaugeRankCacheLength.Set(float64(length)) var removeItems []bitmapPair // cached, ordered list if length > int(c.maxEntries) { @@ -333,7 +320,7 @@ func (c *rankCache) recalculate() { // If size is larger than the threshold then trim it. if len(c.entries) > c.thresholdBuffer { - c.stats.Count(MetricCacheThresholdReached, 1, 1.0) + CounterCacheThresholdReached.Inc() for _, pair := range removeItems { delete(c.entries, pair.ID) } @@ -343,11 +330,6 @@ func (c *rankCache) recalculate() { c.dirty = false } -// SetStats defines the stats client used in the cache. -func (c *rankCache) SetStats(s stats.StatsClient) { - c.stats = s -} - // Top returns an ordered list of pairs. func (c *rankCache) Top() []bitmapPair { c.mu.Lock() @@ -355,7 +337,7 @@ func (c *rankCache) Top() []bitmapPair { if c.dirty { // The cache is dirty, so we need to recalculate it to get a consistent view. - c.stats.Count(MetricReadDirtyCache, 1, 1.0) + CounterReadDirtyCache.Inc() c.recalculate() } @@ -606,25 +588,21 @@ func (p uint64Slice) Len() int { return len(p) } func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } // nopCache represents a no-op Cache implementation. -type nopCache struct { - stats stats.StatsClient -} +type nopCache struct{} // Ensure NopCache implements Cache. -var globalNopCache cache = nopCache{ - stats: stats.NopStatsClient, -} +var globalNopCache cache = nopCache{} func (c nopCache) Add(uint64, uint64) {} func (c nopCache) BulkAdd(uint64, uint64) {} func (c nopCache) Get(uint64) uint64 { return 0 } func (c nopCache) IDs() []uint64 { return []uint64{} } -func (c nopCache) Invalidate() {} -func (c nopCache) Len() int { return 0 } -func (c nopCache) Recalculate() {} -func (c nopCache) SetStats(stats.StatsClient) {} -func (c nopCache) Clear() {} +func (c nopCache) Invalidate() {} +func (c nopCache) Len() int { return 0 } +func (c nopCache) Recalculate() {} + +func (c nopCache) Clear() {} func (c nopCache) Top() []bitmapPair { return []bitmapPair{} diff --git a/client/client.go b/client/client.go index 0b313b971..2dca46206 100644 --- a/client/client.go +++ b/client/client.go @@ -52,7 +52,6 @@ type Client struct { manualFragmentNode *fragmentNode manualServerURI *pnet.URI tracer opentracing.Tracer - Stats stats.StatsClient // An exponential backoff algorithm retries requests exponentially (if an HTTP request fails), // increasing the waiting time between retries up to a maximum backoff time. maxBackoff time.Duration @@ -211,11 +210,6 @@ func newClientWithOptions(options *ClientOptions) *Client { } else { c.tracer = options.tracer } - if options.stats == nil { - c.Stats = stats.NopStatsClient - } else { - c.Stats = options.stats - } c.maxRetries = *options.retries c.maxBackoff = 2 * time.Minute @@ -1359,7 +1353,6 @@ type ClientOptions struct { manualServerAddress bool tracer opentracing.Tracer retries *int - stats stats.StatsClient nat map[pnet.URI]pnet.URI pathPrefix string } @@ -1445,14 +1438,6 @@ func OptClientRetries(retries int) ClientOption { } } -// OptClientStatsClient sets a stats client, such as Prometheus -func OptClientStatsClient(stats stats.StatsClient) ClientOption { - return func(options *ClientOptions) error { - options.stats = stats - return nil - } -} - // OptClientNAT sets a NAT map used to translate the advertised URI to something // else (for example, when accessing pilosa running in docker). func OptClientNAT(nat map[string]string) ClientOption { diff --git a/client/importer.go b/client/importer.go index ae80dee78..27a2cc7ef 100644 --- a/client/importer.go +++ b/client/importer.go @@ -328,7 +328,3 @@ func (i *importer) EncodeImport(ctx context.Context, tid dax.TableID, fld *dax.F func (i *importer) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, path string, data []byte) error { return i.client.DoImport(string(tid), shard, path, data) } - -func (i *importer) StatsTiming(name string, value time.Duration, rate float64) { - i.client.Stats.Timing(name, value, rate) -} diff --git a/dax/README.md b/dax/README.md index 97c18b5e5..c17a52773 100644 --- a/dax/README.md +++ b/dax/README.md @@ -10,7 +10,7 @@ the Controller, as well as FeatureBase and IDK-based ingesters. The DAX test currently requires docker images for: `featurebase` and `datagen`. If at any point you run into problems with go mod failing to reference a private -repo, make sure that you have `gitlab.com/molecula` in your `GOPRIVATE` +repo, make sure that you have `gitlab.com/featurebasedb` in your `GOPRIVATE` environment variable. Note that during the docker image build step, `go mod vendor` is run, which @@ -24,7 +24,7 @@ These may no longer be relevant. I needed to but this in my `~/.profile` file: -```export GOPRIVATE=github.com/molecula,gitlab.com/molecula``` +```export GOPRIVATE=github.com/featurebasedb,gitlab.com/featurebasedb``` And this in my `~/.gitconfig` @@ -41,9 +41,9 @@ Then `make docker` ran successfully. ### Build the FeatureBase docker image - Check out the - [dax](https://github.com/molecula/featurebase/tree/dax) + [dax](https://github.com/featurebasedb/featurebase/tree/dax) branch of the - [featurebase](https://github.com/molecula/featurebase) repository. + [featurebase](https://github.com/featurebasedb/featurebase) repository. - Run `make docker-image-featurebase` to build the docker image - You should now have an image in docker named `dax/featurebase` with the tag `latest`. @@ -57,8 +57,8 @@ Then `make docker` ran successfully. ## Running the tests - Check out the - [dax](https://github.com/molecula/featurebase/tree/dax) + [dax](https://github.com/featurebasedb/featurebase/tree/dax) branch of the - [featurebase](https://github.com/molecula/featurebase) repository. + [featurebase](https://github.com/featurebasedb/featurebase) repository. - Change into the `dax` directory: `cd dax` - Run `make test-integration`. diff --git a/dax/queryer/orchestrator.go b/dax/queryer/orchestrator.go index 84a1979e3..15b28e16e 100644 --- a/dax/queryer/orchestrator.go +++ b/dax/queryer/orchestrator.go @@ -74,7 +74,7 @@ type Translator interface { // TODO(jaffee) the naming here is a cluster. TranslateIndexIDs takes a list, but TranslateFieldIDs takes a set, both have alternate methods that take the other thing. :facepalm: TranslateIndexIDs(ctx context.Context, index string, ids []uint64) ([]string, error) TranslateIndexIDSet(ctx context.Context, index string, ids map[uint64]struct{}) (map[uint64]string, error) - TranslateFieldIDs(ctx context.Context, index, field string, ids map[uint64]struct{}) (map[uint64]string, error) + TranslateFieldIDs(ctx context.Context, tableKeyer dax.TableKeyer, field string, ids map[uint64]struct{}) (map[uint64]string, error) TranslateFieldListIDs(ctx context.Context, index, field string, ids []uint64) ([]string, error) } @@ -87,7 +87,6 @@ type orchestrator struct { // Client used for remote requests. client *featurebase.InternalClient - stats stats.StatsClient logger logger.Logger } @@ -433,11 +432,11 @@ func (o *orchestrator) executeCall(ctx context.Context, tableKeyer dax.TableKeye } else if err := o.validateCallArgs(c); err != nil { return nil, errors.Wrap(err, "validating args") } - indexTag := "index:" + string(tableKeyer.Key()) - metricName := "query_" + strings.ToLower(c.Name) + "_total" - statFn := func() { + + labels := prometheus.Labels{"index": string(tableKeyer.Key())} + statFn := func(ctr *prometheus.CounterVec) { if !opt.Remote { - o.stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag}) + ctr.With(labels).Inc() } } @@ -449,101 +448,106 @@ func (o *orchestrator) executeCall(ctx context.Context, tableKeyer dax.TableKeye switch c.Name { case "Sum": - statFn() + statFn(featurebase.CounterQuerySumTotal) res, err := o.executeSum(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeSum") case "Min": - statFn() + statFn(featurebase.CounterQueryMinTotal) res, err := o.executeMin(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeMin") case "Max": - statFn() + statFn(featurebase.CounterQueryMaxTotal) res, err := o.executeMax(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeMax") case "MinRow": - statFn() + statFn(featurebase.CounterQueryMinRowTotal) res, err := o.executeMinRow(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeMinRow") case "MaxRow": - statFn() + statFn(featurebase.CounterQueryMaxRowTotal) res, err := o.executeMaxRow(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeMaxRow") // case "Clear": - // statFn() + // statFn(featurebase.CounterQueryClearTotal) // res, err := o.executeClearBit(ctx, index, c, opt) // return res, errors.Wrap(err, "executeClearBit") // case "ClearRow": - // statFn() + // statFn(featurebase.CounterQueryClearRowTotal) // res, err := o.executeClearRow(ctx, index, c, shards, opt) // return res, errors.Wrap(err, "executeClearRow") case "Distinct": - statFn() + statFn(featurebase.CounterQueryDistinctTotal) res, err := o.executeDistinct(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeDistinct") // case "Store": - // statFn() + // statFn(featurebase.CounterQueryStoreTotal) // res, err := o.executeSetRow(ctx, index, c, shards, opt) // return res, errors.Wrap(err, "executeSetRow") case "Count": - statFn() + statFn(featurebase.CounterQueryCountTotal) res, err := o.executeCount(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeCount") // case "Set": - // statFn() + // statFn(featurebase.CounterQuerySetTotal) // res, err := o.executeSet(ctx, index, c, opt) // return res, errors.Wrap(err, "executeSet") case "TopK": - statFn() + statFn(featurebase.CounterQueryTopKTotal) res, err := o.executeTopK(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeTopK") case "TopN": - statFn() + statFn(featurebase.CounterQueryTopNTotal) res, err := o.executeTopN(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeTopN") case "Rows": - statFn() + statFn(featurebase.CounterQueryRowsTotal) res, err := o.executeRows(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeRows") case "Extract": - statFn() + statFn(featurebase.CounterQueryExtractTotal) res, err := o.executeExtract(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeExtract") case "GroupBy": - statFn() + statFn(featurebase.CounterQueryGroupByTotal) res, err := o.executeGroupBy(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeGroupBy") case "Options": - statFn() + statFn(featurebase.CounterQueryOptionsTotal) res, err := o.executeOptionsCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeOptionsCall") case "IncludesColumn": + statFn(featurebase.CounterQueryIncludesColumnTotal) res, err := o.executeIncludesColumnCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeIncludesColumnCall") case "FieldValue": - statFn() + statFn(featurebase.CounterQueryFieldValueTotal) res, err := o.executeFieldValueCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeFieldValueCall") case "Precomputed": + statFn(featurebase.CounterQueryPrecomputedTotal) res, err := o.executePrecomputedCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executePrecomputedCall") case "UnionRows": + statFn(featurebase.CounterQueryUnionRowsTotal) res, err := o.executeUnionRows(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeUnionRows") case "ConstRow": + statFn(featurebase.CounterQueryConstRowTotal) res, err := o.executeConstRow(ctx, tableKeyer, c) return res, errors.Wrap(err, "executeConstRow") case "Limit": + statFn(featurebase.CounterQueryLimitTotal) res, err := o.executeLimitCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeLimitCall") case "Percentile": + statFn(featurebase.CounterQueryPercentileTotal) res, err := o.executePercentile(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executePercentile") // case "Delete": - // statFn() //TODO(twg) need this? + // statFn(featurebase.CounterQueryDeleteTotal) // res, err := o.executeDeleteRecords(ctx, index, c, shards, opt) // return res, errors.Wrap(err, "executeDelete") default: // o.g. "Row", "Union", "Intersect" or anything that returns a bitmap. - statFn() res, err := o.executeBitmapCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeBitmapCall") } @@ -1116,13 +1120,42 @@ func (o *orchestrator) executeBitmapCall(ctx context.Context, tableKeyer dax.Tab span.LogKV("pqlCallName", c.Name) defer span.Finish() - indexTag := "index:" + string(tableKeyer.Key()) - metricName := "query_" + strings.ToLower(c.Name) + "_total" - if c.Name == "Row" && c.HasConditionArg() { - metricName = "query_row_bsi_total" + labels := prometheus.Labels{"index": string(tableKeyer.Key())} + statFn := func(ctr *prometheus.CounterVec) { + if !opt.Remote { + ctr.With(labels).Inc() + } } + if !opt.Remote { - o.stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag}) + switch c.Name { + case "Row": + if c.HasConditionArg() { + statFn(featurebase.CounterQueryRowBSITotal) + } else { + statFn(featurebase.CounterQueryRowTotal) + } + case "Range": + statFn(featurebase.CounterQueryRangeTotal) + case "Difference": + statFn(featurebase.CounterQueryBitmapTotal) + case "Intersect": + statFn(featurebase.CounterQueryIntersectTotal) + case "Union": + statFn(featurebase.CounterQueryUnionTotal) + case "InnerUnionRows": + statFn(featurebase.CounterQueryInnerUnionRowsTotal) + case "Xor": + statFn(featurebase.CounterQueryXorTotal) + case "Not": + statFn(featurebase.CounterQueryNotTotal) + case "Shift": + statFn(featurebase.CounterQueryShiftTotal) + case "All": + statFn(featurebase.CounterQueryAllTotal) + default: + statFn(featurebase.CounterQueryBitmapTotal) + } } // Merge returned results at coordinating node. @@ -2985,9 +3018,7 @@ func (o *orchestrator) preTranslateMatrixSet(ctx context.Context, mat featurebas } } - index := string(tableKeyer.Key()) - - return o.trans.TranslateFieldIDs(ctx, index, field, ids) + return o.trans.TranslateFieldIDs(ctx, tableKeyer, field, ids) } func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedTable, call *pql.Call, result interface{}, idSet map[uint64]string) (_ interface{}, err error) { @@ -3164,7 +3195,7 @@ func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedT fieldTranslations := make(map[string]map[uint64]string) for field, ids := range fieldIDs { - trans, err := o.trans.TranslateFieldIDs(ctx, idx.Name, field.Name, ids) + trans, err := o.trans.TranslateFieldIDs(ctx, qtbl, field.Name, ids) if err != nil { return nil, errors.Wrapf(err, "translating IDs in field '%q'", field.Name) } diff --git a/dax/queryer/queryer.go b/dax/queryer/queryer.go index 09b976c5e..1356e8e8d 100644 --- a/dax/queryer/queryer.go +++ b/dax/queryer/queryer.go @@ -89,7 +89,6 @@ func (q *Queryer) Orchestrator(qual dax.TableQualifier) *qualifiedOrchestrator { topology: &MDSTopology{noder: q.noder}, // TODO(jaffee) using default http.Client probably bad... need to set some timeouts. client: q.fbClient, - stats: stats.NopStatsClient, logger: q.logger, } diff --git a/dax/queryer/translator.go b/dax/queryer/translator.go index 90de80b07..024082d56 100644 --- a/dax/queryer/translator.go +++ b/dax/queryer/translator.go @@ -229,15 +229,19 @@ func (m *mdsTranslator) TranslateIndexIDSet(ctx context.Context, table string, i } return ret, nil } -func (m *mdsTranslator) TranslateFieldIDs(ctx context.Context, table, field string, ids map[uint64]struct{}) (map[uint64]string, error) { +func (m *mdsTranslator) TranslateFieldIDs(ctx context.Context, tableKeyer dax.TableKeyer, field string, ids map[uint64]struct{}) (map[uint64]string, error) { idList := make([]uint64, 0, len(ids)) for id := range ids { idList = append(idList, id) } - stringList, err := m.TranslateFieldListIDs(ctx, table, field, idList) + // TODO(tlt): convert TranslateFieldListIDs (and the other Translator + // interface methods) to TableKeyer. + index := string(tableKeyer.Key()) + + stringList, err := m.TranslateFieldListIDs(ctx, index, field, idList) if err != nil { - return nil, errors.Wrapf(err, "translating field ids on field: %s, %s", table, field) + return nil, errors.Wrapf(err, "translating field ids on field: %s, %s", tableKeyer, field) } ret := make(map[uint64]string) diff --git a/dax/test/dax/dax_test.go b/dax/test/dax/dax_test.go index 95ff794c6..a5f026412 100644 --- a/dax/test/dax/dax_test.go +++ b/dax/test/dax/dax_test.go @@ -112,7 +112,6 @@ func TestDAXIntegration(t *testing.T) { "alterTable/alterTableBadTable", // looks like table does not exist is a different error in DAX "top-tests/test-1", // don't know why this is failing at all "delete_tests", - "subquerytable", // subqueries seem to be a problem } doSkip := func(name string) bool { diff --git a/executor.go b/executor.go index 2172f39b3..4fd1023a2 100644 --- a/executor.go +++ b/executor.go @@ -29,6 +29,7 @@ import ( "github.com/featurebasedb/featurebase/v3/testhook" "github.com/featurebasedb/featurebase/v3/tracing" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" ) @@ -165,16 +166,16 @@ func (e *executor) Close() error { // PoolSize is exported to let the task pool update us func (e *executor) PoolSize(n int) { if e.Holder != nil { - e.Holder.Stats.Gauge("worker_total", float64(n), 0) + GaugeWorkerTotal.Set(float64(n)) } } // InitStats initializes stats counters. Must be called after Holder set. func (e *executor) InitStats() { if e.Holder != nil { - e.Holder.Stats.Count("job_total", 0, 0) + CounterJobTotal.Add(0) l, _, _ := e.workers.Stats() - e.Holder.Stats.Gauge("worker_total", float64(l), 0) + GaugeWorkerTotal.Set(float64(l)) } } @@ -684,11 +685,11 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p } else if err := e.validateCallArgs(c); err != nil { return nil, errors.Wrap(err, "validating args") } - indexTag := "index:" + index - metricName := "query_" + strings.ToLower(c.Name) + "_total" - statFn := func() { + + labels := prometheus.Labels{"index": index} + statFn := func(ctr *prometheus.CounterVec) { if !opt.Remote { - e.Holder.Stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag}) + ctr.With(labels).Inc() } } @@ -720,114 +721,122 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p switch c.Name { case "Sum": - statFn() + statFn(CounterQuerySumTotal) res, err := e.executeSum(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeSum") case "Min": - statFn() + statFn(CounterQueryMinTotal) res, err := e.executeMin(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeMin") case "Max": - statFn() + statFn(CounterQueryMaxTotal) res, err := e.executeMax(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeMax") case "MinRow": - statFn() + statFn(CounterQueryMinRowTotal) res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeMinRow") case "MaxRow": - statFn() + statFn(CounterQueryMaxRowTotal) res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeMaxRow") case "Clear": - statFn() + statFn(CounterQueryClearTotal) res, err := e.executeClearBit(ctx, qcx, index, c, opt) return res, errors.Wrap(err, "executeClearBit") case "ClearRow": - statFn() + statFn(CounterQueryClearRowTotal) res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeClearRow") case "Distinct": - statFn() + statFn(CounterQueryDistinctTotal) res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeDistinct") case "Store": - statFn() + statFn(CounterQueryStoreTotal) res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeSetRow") case "Count": - statFn() + statFn(CounterQueryCountTotal) res, err := e.executeCount(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeCount") case "Set": - statFn() + statFn(CounterQuerySetTotal) res, err := e.executeSet(ctx, qcx, index, c, opt) return res, errors.Wrap(err, "executeSet") case "TopK": - statFn() + statFn(CounterQueryTopKTotal) res, err := e.executeTopK(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeTopK") case "TopN": - statFn() + statFn(CounterQueryTopNTotal) res, err := e.executeTopN(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeTopN") case "Rows": - statFn() + statFn(CounterQueryRowsTotal) res, err := e.executeRows(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeRows") case "ExternalLookup": - statFn() + statFn(CounterQueryExternalLookupTotal) res, err := e.executeExternalLookup(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeExternalLookup") case "Extract": - statFn() + statFn(CounterQueryExtractTotal) res, err := e.executeExtract(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeExtract") case "GroupBy": - statFn() + statFn(CounterQueryGroupByTotal) res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeGroupBy") case "Options": - statFn() + statFn(CounterQueryOptionsTotal) res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeOptionsCall") case "IncludesColumn": + statFn(CounterQueryIncludesColumnTotal) res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeIncludesColumnCall") case "FieldValue": - statFn() + statFn(CounterQueryFieldValueTotal) res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeFieldValueCall") case "Precomputed": + statFn(CounterQueryPrecomputedTotal) res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executePrecomputedCall") case "UnionRows": + statFn(CounterQueryUnionRowsTotal) res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeUnionRows") case "ConstRow": + statFn(CounterQueryConstRowTotal) res, err := e.executeConstRow(ctx, index, c) return res, errors.Wrap(err, "executeConstRow") case "Limit": + statFn(CounterQueryLimitTotal) res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeLimitCall") case "Percentile": + statFn(CounterQueryPercentileTotal) res, err := e.executePercentile(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executePercentile") case "Delete": - statFn() // TODO(twg) need this? + statFn(CounterQueryDeleteTotal) res, err := e.executeDeleteRecords(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeDelete") case "Sort": + statFn(CounterQuerySortTotal) res, err := e.executeSort(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeSort") case "Apply": + statFn(CounterQueryApplyTotal) res, err := e.executeApply(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeApply") case "Arrow": + statFn(CounterQueryArrowTotal) res, err := e.executeArrow(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeArrow") default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap. - statFn() res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeBitmapCall") } @@ -1506,13 +1515,42 @@ func (e *executor) executeBitmapCall(ctx context.Context, qcx *Qcx, index string span.LogKV("pqlCallName", c.Name) defer span.Finish() - indexTag := "index:" + index - metricName := "query_" + strings.ToLower(c.Name) + "_total" - if c.Name == "Row" && c.HasConditionArg() { - metricName = "query_row_bsi_total" + labels := prometheus.Labels{"index": index} + statFn := func(ctr *prometheus.CounterVec) { + if !opt.Remote { + ctr.With(labels).Inc() + } } + if !opt.Remote { - e.Holder.Stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag}) + switch c.Name { + case "Row": + if c.HasConditionArg() { + statFn(CounterQueryRowBSITotal) + } else { + statFn(CounterQueryRowTotal) + } + case "Range": + statFn(CounterQueryRangeTotal) + case "Difference": + statFn(CounterQueryBitmapTotal) + case "Intersect": + statFn(CounterQueryIntersectTotal) + case "Union": + statFn(CounterQueryUnionTotal) + case "InnerUnionRows": + statFn(CounterQueryInnerUnionRowsTotal) + case "Xor": + statFn(CounterQueryXorTotal) + case "Not": + statFn(CounterQueryNotTotal) + case "Shift": + statFn(CounterQueryShiftTotal) + case "All": + statFn(CounterQueryAllTotal) + default: + statFn(CounterQueryBitmapTotal) + } } // Execute calls in bulk on each remote node and merge. diff --git a/field.go b/field.go index 94cb7b8e3..bd21f63ea 100644 --- a/field.go +++ b/field.go @@ -84,7 +84,6 @@ type Field struct { viewMap map[string]*view broadcaster broadcaster - Stats stats.StatsClient serializer Serializer // Field options. @@ -396,7 +395,6 @@ func newField(holder *Holder, path, index, name string, opts FieldOption) (*Fiel viewMap: make(map[string]*view), broadcaster: NopBroadcaster, - Stats: stats.NopStatsClient, serializer: NopSerializer, options: applyDefaultOptions(&fo), @@ -1165,7 +1163,6 @@ func (f *Field) newView(path, name string) *view { view := newView(f.holder, path, f.index, f.name, name, f.options) view.idx = f.idx view.fld = f - view.stats = f.Stats view.broadcaster = f.broadcaster return view } diff --git a/fragment.go b/fragment.go index 801a9d2a7..a8731d605 100644 --- a/fragment.go +++ b/fragment.go @@ -120,8 +120,6 @@ type fragment struct { // mutexVector is used for mutex field types. It's checked for an // existing value (to clear) prior to setting a new value. mutexVector vector - - stats stats.StatsClient } // newFragment returns a new instance of fragment. @@ -142,8 +140,6 @@ func newFragment(holder *Holder, idx *Index, fld *Field, vw *view, shard uint64) CacheSize: DefaultCacheSize, holder: holder, - - stats: stats.NopStatsClient, } return f } @@ -405,7 +401,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo f.cache.Add(rowID, n) } - f.stats.Count(MetricSetBit, 1, 1.0) + CounterSetBit.Inc() return changed, nil } @@ -454,7 +450,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b f.cache.Add(rowID, n) } - f.stats.Count(MetricClearBit, 1, 1.0) + CounterClearBit.Inc() return changed, nil } @@ -510,7 +506,7 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo } } - f.stats.Count("setRow", 1, 1.0) + CounterSetRow.Inc() return changed, nil } @@ -1713,23 +1709,23 @@ func (p parallelSlices) Swap(i, j int) { // operations to the op log. func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64]struct{}) error { if len(set) > 0 { - f.stats.Count(MetricImportingN, int64(len(set)), 1) + CounterImportingN.Add(float64(len(set))) // TODO benchmark Add/RemoveN behavior with sorted/unsorted positions changedN, err := tx.Add(f.index(), f.field(), f.view(), f.shard, set...) if err != nil { return errors.Wrap(err, "adding positions") } - f.stats.Count(MetricImportedN, int64(changedN), 1) + CounterImportedN.Add(float64(changedN)) } if len(clear) > 0 { - f.stats.Count(MetricClearingN, int64(len(clear)), 1) + CounterClearingingN.Add(float64(len(clear))) changedN, err := tx.Remove(f.index(), f.field(), f.view(), f.shard, clear...) if err != nil { return errors.Wrap(err, "clearing positions") } - f.stats.Count(MetricClearedN, int64(changedN), 1) + CounterClearedN.Add(float64(changedN)) } return f.updateCaching(tx, rowSet) } diff --git a/go.mod b/go.mod index a7caa917e..341712cb2 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ replace robpike.io/ivy => github.com/tgruben/ivy v0.0.0-20221107170120-634b546dc require ( github.com/CAFxX/gcnotifier v0.0.0-20220409005548-0153238b886a - github.com/DataDog/datadog-go v4.8.3+incompatible + github.com/DataDog/datadog-go v4.8.3+incompatible // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2 diff --git a/holder.go b/holder.go index 89533d2fc..ba079cc9b 100644 --- a/holder.go +++ b/holder.go @@ -80,9 +80,6 @@ type Holder struct { wg sync.WaitGroup closing chan struct{} - // Stats - Stats stats.StatsClient - // Data directory path. path string @@ -257,7 +254,6 @@ type HolderConfig struct { Schemator disco.Schemator Sharder disco.Sharder CacheFlushInterval time.Duration - StatsClient stats.StatsClient Logger logger.Logger StorageConfig *storage.Config @@ -282,7 +278,6 @@ func DefaultHolderConfig() *HolderConfig { Schemator: disco.NewInMemSchemator(), Sharder: disco.InMemSharder, CacheFlushInterval: defaultCacheFlushInterval, - StatsClient: stats.NopStatsClient, Logger: logger.NopLogger, StorageConfig: storage.NewDefaultConfig(), RBFConfig: rbfcfg.NewDefaultConfig(), @@ -324,7 +319,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { broadcaster: NopBroadcaster, partitionN: cfg.PartitionN, - Stats: cfg.StatsClient, cacheFlushInterval: cfg.CacheFlushInterval, OpenTranslateStore: cfg.OpenTranslateStore, OpenTranslateReader: cfg.OpenTranslateReader, @@ -527,8 +521,6 @@ func (h *Holder) Open() error { // Check if deletion was in progress when server was shutdown h.processDeleteInflight() - h.Stats.Open() - h.opened.Close() _ = testhook.Opened(h.Auditor, h, nil) @@ -629,8 +621,6 @@ func (h *Holder) Close() error { fmt.Printf("%v\n", globalCallStats.report()) } - h.Stats.Close() - // Notify goroutines of closing and wait for completion. close(h.closing) h.wg.Wait() @@ -1157,7 +1147,6 @@ func (h *Holder) newIndex(path, name string) (*Index, error) { if err != nil { return nil, err } - index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name())) index.broadcaster = h.broadcaster index.serializer = h.serializer index.OpenTranslateStore = h.OpenTranslateStore @@ -1330,9 +1319,6 @@ type holderSyncer struct { syncers errgroup.Group - // Stats - Stats stats.StatsClient - // Signals that the sync should stop. Closing <-chan struct{} } diff --git a/http_handler.go b/http_handler.go index bbb72a5f2..2bfe07cfd 100644 --- a/http_handler.go +++ b/http_handler.go @@ -9,7 +9,6 @@ import ( "encoding/gob" "encoding/hex" "encoding/json" - "expvar" "fmt" "io" "math" @@ -45,6 +44,7 @@ import ( "github.com/featurebasedb/featurebase/v3/storage" "github.com/featurebasedb/featurebase/v3/tracing" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" dto "github.com/prometheus/client_model/go" "github.com/prometheus/prom2json" @@ -402,8 +402,7 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { next.ServeHTTP(w, r) dur := time.Since(t) - statsTags := make([]string, 0, 5) - + isSlow := "false" longQueryTime := h.api.LongQueryTime() if longQueryTime > 0 && dur > longQueryTime { queryRequest := r.Context().Value(contextKeyQueryRequest) @@ -414,31 +413,27 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { } h.logger.Printf("HTTP query duration %v exceeds %v: %s %s %s", dur, longQueryTime, r.Method, r.URL.String(), queryString) - statsTags = append(statsTags, "slow:true") - } else { - statsTags = append(statsTags, "slow:false") + isSlow = "true" } + where := "" pathParts := strings.Split(r.URL.Path, "/") if externalPrefixFlag[pathParts[1]] { - statsTags = append(statsTags, "where:external") + where = "external" } else { - statsTags = append(statsTags, "where:internal") + where = "internal" } - - statsTags = append(statsTags, "useragent:"+r.UserAgent()) - path, err := mux.CurrentRoute(r).GetPathTemplate() - if err == nil { - statsTags = append(statsTags, "path:"+path) - } - - statsTags = append(statsTags, "method:"+r.Method) - - stats := h.api.StatsWithTags(statsTags) - if stats != nil { - stats.Timing(MetricHTTPRequest, dur, 0.1) + if err != nil { + path = "" } + SummaryHttpRequests.With(prometheus.Labels{ + "method": r.Method, + "path": path, + "slow": isSlow, + "useragent": r.UserAgent(), + "where": where, + }).Observe(dur.Seconds()) }) } @@ -506,7 +501,6 @@ func newRouter(handler *Handler) http.Handler { // TODO: figure out how to protect these if needed router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.PathPrefix("/debug/fgprof").Handler(fgprof.Handler()).Methods("GET") - router.Handle("/debug/vars", expvar.Handler()).Methods("GET") router.Handle("/metrics", promhttp.Handler()) router.HandleFunc("/metrics.json", handler.chkAuthZ(handler.handleGetMetricsJSON, authz.Admin)).Methods("GET").Name("GetMetricsJSON") @@ -552,6 +546,8 @@ func newRouter(handler *Handler) http.Handler { if handler.sqlEnabled { router.HandleFunc("/sql", handler.chkAuthZ(handler.handlePostSQL, authz.Admin)).Methods("POST").Name("PostSQL") } + // internal endpoint + router.HandleFunc("/sql", handler.chkAuthZ(handler.handlePostSQLPlanOperator, authz.Admin)).Headers("X-FeatureBase-Plan-Operator", "").Methods("POST").Name("PostSQLPlanOperator") router.HandleFunc("/query-history", handler.chkAuthZ(handler.handleGetPastQueries, authz.Admin)).Methods("GET").Name("GetPastQueries") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion") @@ -1393,10 +1389,70 @@ func (h *Handler) writeBadRequest(w http.ResponseWriter, r *http.Request, err er } } -// handlePostSQL handles /sql requests. -func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { - includePlan := false +// handlePostSQLOperator handles an internal sql3 plan operator execution request +// these requests come from other nodes in the cluster +// handlePostSQLOperator will 'rehydrate' a plan operator and return data in the +// featurebase wire format for effciency +// we do not track these requests as user requests +// TODO(pok) - thus is there anything we need here to align with how we do this for other nodes +func (h *Handler) handlePostSQLPlanOperator(w http.ResponseWriter, r *http.Request) { + writeError := func(err error) { + if err != nil { + w.Write(wireprotocol.WriteError(err)) + } + } + + // always finish with a done message + defer w.Write(wireprotocol.WriteDone()) + + ctx := r.Context() + + rootOperator, err := h.api.RehydratePlanOperator(ctx, r.Body) + if err != nil { + writeError(err) + return + } + + // get a query iterator. + iter, err := rootOperator.Iterator(ctx, nil) + if err != nil { + writeError(err) + return + } + // read schema & write to response. + columns := rootOperator.Schema() + b, err := wireprotocol.WriteSchema(columns) + if err != nil { + writeError(err) + return + } + w.Write(b) + + var rowErr error + var currentRow types.Row + var nextErr error + + for currentRow, nextErr = iter.Next(ctx); nextErr == nil; currentRow, nextErr = iter.Next(ctx) { + b, err := wireprotocol.WriteRow(currentRow, columns) + if err != nil { + rowErr = err + break + } + w.Write(b) + } + if nextErr != nil && nextErr != types.ErrNoMoreRows { + rowErr = nextErr + } + writeError(rowErr) +} + +// handlePostSQL handles /sql requests +// supports a ?plan=true|false parameter to send back the plan in the +// query response +func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { + + includePlan := false includePlanValue := r.URL.Query().Get("plan") if len(includePlanValue) > 0 { var err error @@ -1407,6 +1463,7 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { } } + // get the body b, err := io.ReadAll(r.Body) if err != nil { h.writeBadRequest(w, r, err) @@ -1420,6 +1477,9 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { // put the requestId in the context ctx := fbcontext.WithRequestID(r.Context(), requestID.String()) + // update the counter for requests + PerfCounterSQLRequestSec.Add(1) + // Write response back to client. w.Header().Set("Content-Type", "application/json") diff --git a/idk/Makefile b/idk/Makefile index cac7251b3..8c520af2d 100644 --- a/idk/Makefile +++ b/idk/Makefile @@ -231,7 +231,7 @@ TPKG ?= ./... test-run: testenv vendor $(DOCKER_COMPOSE) build idk-test $(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v -mod=vendor -tags=odbc,dynamic $(TPKG) -covermode=atomic -coverpkg=$(TPKG) -coverprofile=/testdata/$(PROJECT)_base_coverage.out" - $(DOCKER_COMPOSE) run -T idk-test /go/src/github.com/molecula/featurebase/idk/reingest_test.sh + $(DOCKER_COMPOSE) run -T idk-test /go/src/github.com/featurebasedb/featurebase/idk/reingest_test.sh test-run-race: testenv vendor diff --git a/idk/ingest.go b/idk/ingest.go index 7cf88f783..47d33280d 100644 --- a/idk/ingest.go +++ b/idk/ingest.go @@ -55,7 +55,6 @@ const ( ) // TODO Jaeger -// TODO Prometheus // Main holds all config for general ingest type Main struct { @@ -132,7 +131,6 @@ type Main struct { newNexter func(c int) (IDAllocator, error) ra RangeAllocator - stats stats.StatsClient metricsServer *http.Server log logger.Logger @@ -218,7 +216,7 @@ func NewMain() *Main { Concurrency: 1, CacheLength: 64, PackBools: "bools", - Namespace: "ingester", + Namespace: "ingester", // this is now ignored and hardcoded in metrics.go IDAllocKeyPrefix: "ingest", UseShardTransactionalEndpoint: os.Getenv("IDK_DEFAULT_SHARD_TRANSACTIONAL") != "", @@ -228,8 +226,6 @@ func NewMain() *Main { SchemaManager: NopSchemaManager, - stats: stats.NopStatsClient, - log: logger.NewStandardLogger(os.Stderr), } } @@ -457,7 +453,6 @@ initialFetch: if v, ok := source.(Metadata); ok { m.log.Printf("new schema - subject: %#v; version: %d; schema: %#v", v.SchemaSubject(), v.SchemaVersion(), v.SchemaSchema()) - // m.log.Printf("new schema: %#v", v.SchemaMetadata()) } else { m.log.Printf("new schema: %#v", schema) } @@ -465,7 +460,7 @@ initialFetch: if err != nil { return errors.Wrap(err, "batchFromSchema") } - m.stats.Count(MetricIngesterSchemaChanges, 1, 1) + CounterIngesterSchemaChanges.Inc() csvSlice = make([]string, len(schema)) if m.csvWriter != nil { for i := range schema { @@ -571,7 +566,7 @@ initialFetch: // skip bad rows only if !rowHasError { err = batch.Add(*row) - m.stats.Count(MetricIngesterRowsAdded, 1, 1) + CounterIngesterRowsAdded.Inc() } if err == pilosabatch.ErrBatchNowFull || err == pilosabatch.ErrBatchNowStale { @@ -958,7 +953,7 @@ func (m *Main) commitRecord(ctx context.Context, rec Record, limitCounter *msgCo return errors.Wrap(err, "committing") } limitCounter.Increment(numRecords) - m.stats.Count(MetricCommittedRecords, int64(numRecords), 1) + CounterCommittedRecords.Add(float64(numRecords)) return nil } @@ -972,7 +967,7 @@ func (m *Main) NewLookupClient() (*PostgresClient, error) { func (m *Main) setupClient() (*tls.Config, error) { var tlsConfig *tls.Config var err error - var opts = []pilosaclient.ClientOption{pilosaclient.OptClientStatsClient(m.stats)} + var opts = []pilosaclient.ClientOption{} if m.TLS.CertificatePath != "" { tlsConfig, err = GetTLSConfig(&m.TLS, m.Log()) if err != nil { @@ -1037,34 +1032,24 @@ func (m *Main) setupClient() (*tls.Config, error) { } func (m *Main) setupStats() error { - if m.Stats != "" { - opts := []prometheus.ClientOption{prometheus.OptClientNamespace(m.Namespace)} - m.stats, _ = prometheus.NewPrometheusClient(opts...) // ignore error that must be nil - - mux := http.NewServeMux() - // reg := prom.NewRegistry() // TODO switch to this once pilosa PrometheusClient is fixed and doesn't use the global registry internally. - // also change prom.DefaultGatherer to be "reg" at that time - reg := prom.DefaultRegisterer - promHandler := promhttp.InstrumentMetricHandler(reg, promhttp.HandlerFor(prom.DefaultGatherer, promhttp.HandlerOpts{})) - mux.Handle("/metrics", promHandler) - - mux.Handle("/metrics.json", metricsJSONHandler{metricsURI: "http://" + m.Stats + "/metrics"}) - m.metricsServer = &http.Server{Addr: m.Stats, Handler: mux} - ln, err := net.Listen("tcp", m.Stats) - if err != nil { - return errors.Wrapf(err, "listen for metrics on '%s'", m.Stats) - } - - go func() { - m.log.Printf("Serving Prometheus metrics with namespace \"%s\" at %v/metrics\n", m.Namespace, m.Stats) - err = m.metricsServer.Serve(ln) - if err != http.ErrServerClosed { - m.log.Printf("serve metrics on '%s': %v", m.Stats, err) - } - }() + mux := http.NewServeMux() + promHandler := promhttp.InstrumentMetricHandler(prom.DefaultRegisterer, promhttp.HandlerFor(prom.DefaultGatherer, promhttp.HandlerOpts{})) + mux.Handle("/metrics", promHandler) + mux.Handle("/metrics.json", metricsJSONHandler{metricsURI: "http://" + m.Stats + "/metrics"}) + m.metricsServer = &http.Server{Addr: m.Stats, Handler: mux} + ln, err := net.Listen("tcp", m.Stats) + if err != nil { + return errors.Wrapf(err, "listen for metrics on '%s'", m.Stats) } - return nil + go func() { + m.log.Printf("Serving Prometheus metrics with namespace \"%s\" at %v/metrics\n", m.Namespace, m.Stats) + err = m.metricsServer.Serve(ln) + if err != http.ErrServerClosed { + m.log.Printf("serve metrics on '%s': %v", m.Stats, err) + } + }() + return nil } type metricsJSONHandler struct { @@ -1218,7 +1203,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error { if err != nil { return errors.Wrap(err, "clearing bools") } - m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:packed-bool"}) + CounterDeleterRowsAdded.With(prom.Labels{"type": "packed-bool"}).Inc() continue } else { fieldName = directive @@ -1267,7 +1252,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error { if err != nil { return errors.Wrap(err, "clearing set") } - m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:set"}) + CounterDeleterRowsAdded.With(prom.Labels{"type": "set"}).Inc() case pilosaclient.FieldTypeMutex: if val == "" { continue @@ -1278,7 +1263,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error { if err != nil { return errors.Wrap(err, "clearing mutex") } - m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:mutex"}) + CounterDeleterRowsAdded.With(prom.Labels{"type": "mutex"}).Inc() case pilosaclient.FieldTypeBool: _, err := client.Query(index.BatchQuery( field.Clear(0, recordID), @@ -1287,19 +1272,19 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error { if err != nil { return errors.Wrap(err, "clearing bool") } - m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:bool"}) + CounterDeleterRowsAdded.With(prom.Labels{"type": "bool"}).Inc() case pilosaclient.FieldTypeInt: _, err := client.Query(field.Clear(0, recordID)) if err != nil { return errors.Wrap(err, "clearing int") } - m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:int"}) + CounterDeleterRowsAdded.With(prom.Labels{"type": "int"}).Inc() case pilosaclient.FieldTypeDecimal: _, err := client.Query(field.Clear(0, recordID)) if err != nil { return errors.Wrap(err, "clearing decimal") } - m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:decimal"}) + CounterDeleterRowsAdded.With(prom.Labels{"type": "decimal"}).Inc() case pilosaclient.FieldTypeTime: return errors.Errorf("deletion on time fields unimplemented") default: diff --git a/idk/ingest_test.go b/idk/ingest_test.go index 7922b8c7a..7a25000f2 100644 --- a/idk/ingest_test.go +++ b/idk/ingest_test.go @@ -525,7 +525,7 @@ func TestIngesterServesPrometheusEndpoint(t *testing.T) { if err != nil { t.Errorf("read error: %v", err) } - if strings.Contains(string(contents), MetricIngesterRowsAdded) { + if !strings.Contains(string(contents), MetricIngesterRowsAdded) { t.Errorf("metric name missing: %v", MetricIngesterRowsAdded) } close(records) diff --git a/idk/mds/importer.go b/idk/mds/importer.go index 6548eca8e..f70c96c34 100644 --- a/idk/mds/importer.go +++ b/idk/mds/importer.go @@ -248,8 +248,6 @@ func (m *importer) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field return fbClient.DoImport(string(qtbl.Key()), shard, path, data) } -func (m *importer) StatsTiming(name string, value time.Duration, rate float64) {} - // getQtbl takes a table (TableKey) and sets the local m.qtbl value. When we // originally set up this type, it was only used by IDK, and the table was known // at the beginning of the process, so it could be set on this import. But diff --git a/idk/metrics.go b/idk/metrics.go index e7d0feabe..cdaac3b33 100644 --- a/idk/metrics.go +++ b/idk/metrics.go @@ -1,8 +1,52 @@ package idk +import "github.com/prometheus/client_golang/prometheus" + const ( MetricDeleterRowsAdded = "deleter_rows_added_total" MetricIngesterRowsAdded = "ingester_rows_added_total" MetricIngesterSchemaChanges = "ingester_schema_changes_total" MetricCommittedRecords = "committed_records" ) + +var CounterIngesterSchemaChanges = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "ingester", + Name: MetricIngesterSchemaChanges, + Help: "TODO", + }, +) + +var CounterIngesterRowsAdded = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "ingester", + Name: MetricIngesterRowsAdded, + Help: "TODO", + }, +) + +var CounterCommittedRecords = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "ingester", + Name: MetricCommittedRecords, + Help: "TODO", + }, +) + +var CounterDeleterRowsAdded = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "ingester", + Name: MetricDeleterRowsAdded, + Help: "TODO", + }, + []string{ + "type", + }, +) + +func init() { + prometheus.MustRegister(CounterIngesterSchemaChanges) + prometheus.MustRegister(CounterIngesterRowsAdded) + prometheus.MustRegister(CounterCommittedRecords) + prometheus.MustRegister(CounterDeleterRowsAdded) +} diff --git a/importer.go b/importer.go index 059b12d42..34d3aac8f 100644 --- a/importer.go +++ b/importer.go @@ -20,8 +20,6 @@ type Importer interface { EncodeImportValues(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, vals []int64, ids []uint64, clear bool) (path string, data []byte, err error) EncodeImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, vals, ids []uint64, clear bool) (path string, data []byte, err error) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, path string, data []byte) error - - StatsTiming(name string, value time.Duration, rate float64) } // Ensure type implements interface. diff --git a/index.go b/index.go index 33078bca0..6914f7197 100644 --- a/index.go +++ b/index.go @@ -20,6 +20,7 @@ import ( "github.com/featurebasedb/featurebase/v3/stats" "github.com/featurebasedb/featurebase/v3/testhook" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" ) @@ -44,7 +45,6 @@ type Index struct { broadcaster broadcaster serializer Serializer - Stats stats.StatsClient // Passed to field for foreign-index lookup. holder *Holder @@ -83,7 +83,6 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) { fields: make(map[string]*Field), broadcaster: NopBroadcaster, - Stats: stats.NopStatsClient, holder: holder, trackExistence: true, @@ -511,7 +510,7 @@ func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap { b.UnionInPlace(f.AvailableShards(localOnly)) } - i.Stats.Gauge(MetricMaxShard, float64(b.Max()), 1.0) + GaugeIndexMaxShard.With(prometheus.Labels{"index": i.name}).Set(float64(b.Max())) return b } @@ -924,7 +923,6 @@ func (i *Index) newField(path, name string) (*Field, error) { return nil, err } f.idx = i - f.Stats = i.Stats f.broadcaster = i.broadcaster f.serializer = i.serializer f.OpenTranslateStore = i.OpenTranslateStore diff --git a/lattice/README.md b/lattice/README.md index 92a20641a..17aa77204 100644 --- a/lattice/README.md +++ b/lattice/README.md @@ -19,4 +19,4 @@ Open [http://localhost:3000](http://localhost:3000) to view it in the browser. Lattice can be embedded within the Pilosa binary, so the UI is fully accessible directly from the server, reducing operational complexity. -If additional build dependencies `yarn` (`brew install yarn` and `brew upgrade yarn` perhaps) and `statik` (`make install-statik`) are available on your system, running `make generate-statik` before `make install` should produce a Pilosa binary with Lattice embedded. For up to date instructions, check the Pilosa [README](https://github.com/molecula/pilosa#getting-started). +If additional build dependencies `yarn` (`brew install yarn` and `brew upgrade yarn` perhaps) and `statik` (`make install-statik`) are available on your system, running `make generate-statik` before `make install` should produce a Pilosa binary with Lattice embedded. For up to date instructions, check the Featurebase [README](https://github.com/featurebasedb/featurebase#getting-started). diff --git a/metrics.go b/metrics.go index 237bb2fa7..249fbc19d 100644 --- a/metrics.go +++ b/metrics.go @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 package pilosa +import "github.com/prometheus/client_golang/prometheus" + const ( MetricCreateIndex = "create_index_total" MetricDeleteIndex = "delete_index_total" @@ -53,3 +55,997 @@ const ( MetricSqlQueries = "sql_queries_total" MetricDeleteDataframe = "delete_dataframe" ) + +const ( + // MetricBatchImportDurationSeconds records the full time of the + // RecordBatch.Import call. This includes starting and finishing a + // transaction, doing key translation, building fragments locally, + // importing all data, and resetting internal structures. + MetricBatchImportDurationSeconds = "batch_import_duration_seconds" + + // MetricBatchFlushDurationSeconds records the full time for + // RecordBatch.Flush (if splitBatchMode is in use). This includes + // starting and finishing a transaction, importing all data, and + // resetting internal structures. + MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds" + + // MetricBatchShardImportBuildRequestsSeconds is the time it takes + // after making fragments to build the shard-transactional request + // objects (but not actually import them or do any network activity). + MetricBatchShardImportBuildRequestsSeconds = "batch_shard_import_build_requests_seconds" + + // MetricBatchShardImportDurationSeconds is the time it takes to + // import all data for all shards in the batch using the + // shard-transactional endpoint. This does not include the time it + // takes to build the requests locally. + MetricBatchShardImportDurationSeconds = "batch_shard_import_duration_seconds" +) + +// server related + +var CounterJobTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "job_total", + Help: "TODO", + }, +) + +var GaugeWorkerTotal = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: "worker_total", + Help: "TODO", + }, +) + +var CounterPQLQueries = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricPqlQueries, + Help: "TODO", + }, +) + +var CounterSQLQueries = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricSqlQueries, + Help: "TODO", + }, +) + +var CounterGarbageCollection = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricGarbageCollection, + Help: "TODO", + }, +) + +var GaugeGoroutines = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricGoroutines, + Help: "TODO", + }, +) + +var GaugeOpenFiles = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricOpenFiles, + Help: "TODO", + }, +) + +var GaugeHeapAlloc = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricHeapAlloc, + Help: "TODO", + }, +) + +var GaugeHeapInUse = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricHeapInuse, + Help: "TODO", + }, +) + +var GaugeStackInUse = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricStackInuse, + Help: "TODO", + }, +) + +var GaugeMallocs = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricMallocs, + Help: "TODO", + }, +) + +var GaugeFrees = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricFrees, + Help: "TODO", + }, +) + +var SummaryHttpRequests = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricHTTPRequest, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, + []string{ + "method", + "path", + "slow", + "useragent", + "where", + }, +) + +var CounterCreateIndex = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricCreateIndex, + Help: "TODO", + }, +) + +var CounterDeleteIndex = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricDeleteIndex, + Help: "TODO", + }, +) + +var CounterDeleteDataframe = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricDeleteDataframe, + Help: "TODO", + }, +) + +var CounterCreateField = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricCreateField, + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterDeleteField = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricDeleteField, + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterDeleteAvailableShard = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricDeleteAvailableShard, + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterExclusiveTransactionRequest = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricExclusiveTransactionRequest, + Help: "TODO", + }, +) + +var CounterTransactionStart = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricTransactionStart, + Help: "TODO", + }, +) + +var CounterExclusiveTransactionBlocked = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricExclusiveTransactionBlocked, + Help: "TODO", + }, +) + +var CounterTransactionBlocked = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricTransactionBlocked, + Help: "TODO", + }, +) + +var CounterExclusiveTransactionActive = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricExclusiveTransactionActive, + Help: "TODO", + }, +) + +var CounterExclusiveTransactionEnd = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricExclusiveTransactionEnd, + Help: "TODO", + }, +) + +var CounterTransactionEnd = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricTransactionEnd, + Help: "TODO", + }, +) + +// TODO(pok) do these need index names? +var CounterRecalculateCache = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricRecalculateCache, + Help: "TODO", + }, +) + +var CounterInvalidateCacheSkipped = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricInvalidateCacheSkipped, + Help: "TODO", + }, +) + +var CounterInvalidateCache = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricInvalidateCache, + Help: "TODO", + }, +) + +var GaugeRankCacheLength = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricRankCacheLength, + Help: "TODO", + }, +) + +var CounterCacheThresholdReached = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricCacheThresholdReached, + Help: "TODO", + }, +) + +var CounterReadDirtyCache = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricReadDirtyCache, + Help: "TODO", + }, +) + +var CounterSetBit = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricSetBit, + Help: "TODO", + }, +) + +var CounterClearBit = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricClearBit, + Help: "TODO", + }, +) + +var CounterSetRow = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "setRow", + Help: "TODO", + }, +) + +var CounterImportingN = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricImportingN, + Help: "TODO", + }, +) + +var CounterImportedN = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricImportedN, + Help: "TODO", + }, +) + +var CounterClearingingN = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricClearingN, + Help: "TODO", + }, +) + +var CounterClearedN = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricClearedN, + Help: "TODO", + }, +) + +var SummaryGRPCStreamQueryDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricGRPCStreamQueryDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryGRPCStreamFormatDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricGRPCStreamFormatDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryGRPCUnaryQueryDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricGRPCUnaryQueryDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryGRPCUnaryFormatDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricGRPCUnaryFormatDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryBatchImportDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricBatchImportDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryBatchFlushDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricBatchFlushDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryBatchShardImportBuildRequestsSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricBatchShardImportBuildRequestsSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryBatchShardImportDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricBatchShardImportDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +// index pql call related + +var CounterQuerySumTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_sum_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryMinTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_min_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryMaxTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_max_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryMinRowTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_minrow_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryMaxRowTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_maxrow_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryClearTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_clear_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryClearRowTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_clearrow_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryDistinctTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_distinct_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryStoreTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_store_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryCountTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_count_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQuerySetTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_set_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryTopKTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_topk_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryTopNTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_topn_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryRowsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_rows_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryExternalLookupTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_externallookup_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryExtractTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_extract_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryGroupByTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_groupby_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryOptionsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_options_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryIncludesColumnTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_includescolumn_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryFieldValueTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_fieldvalue_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryPrecomputedTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_precomputed_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryUnionRowsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_unionrows_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryConstRowTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_constrow_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryLimitTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_limit_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryPercentileTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_percentile_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryDeleteTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_delete_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQuerySortTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_sort_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryApplyTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_apply_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryArrowTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_arrow_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +// bitmap calls +var CounterQueryBitmapTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_bitmap_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryRowTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_row_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryRowBSITotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_row_bsi_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryRangeTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_range_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryDifferenceTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_difference_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryIntersectTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_intersect_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryUnionTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_union_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryInnerUnionRowsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_innerunionrows_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryXorTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_xor_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryNotTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_not_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryShiftTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_shift_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryAllTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_all_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +// index related + +var GaugeIndexMaxShard = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricMaxShard, + Help: "TODO", + }, + []string{ + "index", + }, +) + +func init() { + // server related + prometheus.MustRegister(CounterJobTotal) + prometheus.MustRegister(GaugeWorkerTotal) + prometheus.MustRegister(CounterPQLQueries) + prometheus.MustRegister(CounterSQLQueries) + prometheus.MustRegister(CounterGarbageCollection) + prometheus.MustRegister(GaugeGoroutines) + prometheus.MustRegister(GaugeOpenFiles) + prometheus.MustRegister(GaugeHeapAlloc) + prometheus.MustRegister(GaugeHeapInUse) + prometheus.MustRegister(GaugeStackInUse) + prometheus.MustRegister(GaugeMallocs) + prometheus.MustRegister(GaugeFrees) + prometheus.MustRegister(SummaryHttpRequests) + prometheus.MustRegister(CounterCreateIndex) + prometheus.MustRegister(CounterDeleteIndex) + prometheus.MustRegister(CounterCreateField) + prometheus.MustRegister(CounterDeleteField) + prometheus.MustRegister(CounterDeleteAvailableShard) + prometheus.MustRegister(CounterDeleteDataframe) + prometheus.MustRegister(CounterExclusiveTransactionRequest) + prometheus.MustRegister(CounterTransactionStart) + prometheus.MustRegister(CounterExclusiveTransactionBlocked) + prometheus.MustRegister(CounterTransactionBlocked) + prometheus.MustRegister(CounterExclusiveTransactionActive) + prometheus.MustRegister(CounterExclusiveTransactionEnd) + prometheus.MustRegister(CounterTransactionEnd) + prometheus.MustRegister(CounterRecalculateCache) + prometheus.MustRegister(CounterInvalidateCacheSkipped) + prometheus.MustRegister(CounterInvalidateCache) + prometheus.MustRegister(GaugeRankCacheLength) + prometheus.MustRegister(CounterCacheThresholdReached) + prometheus.MustRegister(CounterReadDirtyCache) + prometheus.MustRegister(CounterSetBit) + prometheus.MustRegister(CounterClearBit) + prometheus.MustRegister(CounterSetRow) + prometheus.MustRegister(CounterImportingN) + prometheus.MustRegister(CounterImportedN) + prometheus.MustRegister(CounterClearingingN) + prometheus.MustRegister(CounterClearedN) + prometheus.MustRegister(SummaryGRPCStreamQueryDurationSeconds) + prometheus.MustRegister(SummaryGRPCStreamFormatDurationSeconds) + prometheus.MustRegister(SummaryGRPCUnaryQueryDurationSeconds) + prometheus.MustRegister(SummaryGRPCUnaryFormatDurationSeconds) + prometheus.MustRegister(SummaryBatchImportDurationSeconds) + prometheus.MustRegister(SummaryBatchFlushDurationSeconds) + prometheus.MustRegister(SummaryBatchShardImportBuildRequestsSeconds) + prometheus.MustRegister(SummaryBatchShardImportDurationSeconds) + + // pql calls + prometheus.MustRegister(CounterQuerySumTotal) + prometheus.MustRegister(CounterQueryMinTotal) + prometheus.MustRegister(CounterQueryMaxTotal) + prometheus.MustRegister(CounterQueryMinRowTotal) + prometheus.MustRegister(CounterQueryMaxRowTotal) + prometheus.MustRegister(CounterQueryClearTotal) + prometheus.MustRegister(CounterQueryClearRowTotal) + prometheus.MustRegister(CounterQueryDistinctTotal) + prometheus.MustRegister(CounterQueryStoreTotal) + prometheus.MustRegister(CounterQueryCountTotal) + prometheus.MustRegister(CounterQuerySetTotal) + prometheus.MustRegister(CounterQueryTopKTotal) + prometheus.MustRegister(CounterQueryTopNTotal) + prometheus.MustRegister(CounterQueryRowsTotal) + prometheus.MustRegister(CounterQueryExternalLookupTotal) + prometheus.MustRegister(CounterQueryExtractTotal) + prometheus.MustRegister(CounterQueryGroupByTotal) + prometheus.MustRegister(CounterQueryOptionsTotal) + prometheus.MustRegister(CounterQueryIncludesColumnTotal) + prometheus.MustRegister(CounterQueryFieldValueTotal) + prometheus.MustRegister(CounterQueryPrecomputedTotal) + prometheus.MustRegister(CounterQueryUnionRowsTotal) + prometheus.MustRegister(CounterQueryConstRowTotal) + prometheus.MustRegister(CounterQueryLimitTotal) + prometheus.MustRegister(CounterQueryPercentileTotal) + prometheus.MustRegister(CounterQueryDeleteTotal) + prometheus.MustRegister(CounterQuerySortTotal) + prometheus.MustRegister(CounterQueryApplyTotal) + prometheus.MustRegister(CounterQueryArrowTotal) + prometheus.MustRegister(CounterQueryBitmapTotal) + prometheus.MustRegister(CounterQueryRowTotal) + prometheus.MustRegister(CounterQueryRowBSITotal) + prometheus.MustRegister(CounterQueryRangeTotal) + prometheus.MustRegister(CounterQueryDifferenceTotal) + prometheus.MustRegister(CounterQueryIntersectTotal) + prometheus.MustRegister(CounterQueryUnionTotal) + prometheus.MustRegister(CounterQueryInnerUnionRowsTotal) + prometheus.MustRegister(CounterQueryXorTotal) + prometheus.MustRegister(CounterQueryNotTotal) + prometheus.MustRegister(CounterQueryShiftTotal) + prometheus.MustRegister(CounterQueryAllTotal) + + // index related + prometheus.MustRegister(GaugeIndexMaxShard) + +} diff --git a/performancecounters.go b/performancecounters.go new file mode 100644 index 000000000..68fcad729 --- /dev/null +++ b/performancecounters.go @@ -0,0 +1,203 @@ +package pilosa + +import ( + "sync/atomic" + + "github.com/prometheus/client_golang/prometheus" +) + +// PerformanceCounter holds data about a performance counter for external consumers +type PerformanceCounter struct { + NameSpace string + SubSystem string + CounterName string + Help string + Value int64 + CounterType int64 +} + +// constants for the counter types +const ( + // raw - for when you just want a count of something + CTR_TYPE_RAW = 0 + // per second - for when you accumulate counts of things + // a consumer would sample this at intervals to arrive at a delta + // then divide by the time in seconds between the samples to get a + // per-second value + CTR_TYPE_PER_SECOND = 1 + // ratio - for when you accumulate a count of something that you + // want to use as a numerator in a ratio calculation + // e.g. 'cache hits' could be a counter of this type and you could + // divide it by a 'cache lookups' counter to get the hit ratio (see below) + CTR_TYPE_RATIO = 2 + // ratio base - for when you accumulate a count of something that you + // want to use as a denominator in a ratio calculation + // e.g. 'cache lookups' could be a counter of this type and you could + // use it as the denominator in a division with a 'cache hits' counter + // as the numerator to get the hit ratio + CTR_TYPE_RATIO_BASE = 3 +) + +type PerformanceCounters struct { + counterValues [5]perfCtrWrapper +} + +type perfCtr struct { + nameSpace string + subSystem string + name string + help string + value int64 + counterType int64 +} + +func (p *perfCtr) Add(increment int64) { + atomic.AddInt64(&p.value, increment) +} + +type perfCtrWrapper struct { + ctr *perfCtr + fun prometheus.CounterFunc +} + +var PerfCounterSQLRequestSec = perfCtr{ + nameSpace: "pilosa", + subSystem: "sql_statistics", + name: "sql_requests_sec", + help: "TODO", + value: 0, + counterType: CTR_TYPE_PER_SECOND, +} + +var PerfCounterSQLInsertsSec = perfCtr{ + nameSpace: "pilosa", + subSystem: "sql_statistics", + name: "sql_inserts_sec", + help: "TODO", + value: 0, + counterType: CTR_TYPE_PER_SECOND, +} + +var PerfCounterSQLBulkInsertsSec = perfCtr{ + nameSpace: "pilosa", + subSystem: "sql_statistics", + name: "sql_bulk_inserts_sec", + help: "TODO", + value: 0, + counterType: CTR_TYPE_PER_SECOND, +} + +var PerfCounterSQLBulkInsertBatchesSec = perfCtr{ + nameSpace: "pilosa", + subSystem: "sql_statistics", + name: "sql_bulk_insert_batches_sec", + help: "TODO", + value: 0, + counterType: CTR_TYPE_PER_SECOND, +} + +var PerfCounterSQLDeletesSec = perfCtr{ + nameSpace: "pilosa", + subSystem: "sql_statistics", + name: "sql_deletes_sec", + help: "TODO", + value: 0, + counterType: CTR_TYPE_PER_SECOND, +} + +var PerfCounters *PerformanceCounters = newPerformanceCounters() + +func newPerformanceCounters() *PerformanceCounters { + ctrs := &PerformanceCounters{ + counterValues: [5]perfCtrWrapper{ + { + &PerfCounterSQLRequestSec, + prometheus.NewCounterFunc( + prometheus.CounterOpts{ + Namespace: PerfCounterSQLRequestSec.nameSpace, + Subsystem: PerfCounterSQLRequestSec.subSystem, + Name: PerfCounterSQLRequestSec.name, + Help: PerfCounterSQLRequestSec.help, + }, + func() float64 { + return float64(atomic.LoadInt64(&PerfCounterSQLRequestSec.value)) + }), + }, + { + &PerfCounterSQLInsertsSec, + prometheus.NewCounterFunc( + prometheus.CounterOpts{ + Namespace: PerfCounterSQLInsertsSec.nameSpace, + Subsystem: PerfCounterSQLInsertsSec.subSystem, + Name: PerfCounterSQLInsertsSec.name, + Help: PerfCounterSQLInsertsSec.help, + }, + func() float64 { + return float64(atomic.LoadInt64(&PerfCounterSQLInsertsSec.value)) + }), + }, + { + &PerfCounterSQLBulkInsertsSec, + prometheus.NewCounterFunc( + prometheus.CounterOpts{ + Namespace: PerfCounterSQLBulkInsertsSec.nameSpace, + Subsystem: PerfCounterSQLBulkInsertsSec.subSystem, + Name: PerfCounterSQLBulkInsertsSec.name, + Help: PerfCounterSQLBulkInsertsSec.help, + }, + func() float64 { + return float64(atomic.LoadInt64(&PerfCounterSQLBulkInsertsSec.value)) + }), + }, + { + &PerfCounterSQLBulkInsertBatchesSec, + prometheus.NewCounterFunc( + prometheus.CounterOpts{ + Namespace: PerfCounterSQLBulkInsertBatchesSec.nameSpace, + Subsystem: PerfCounterSQLBulkInsertBatchesSec.subSystem, + Name: PerfCounterSQLBulkInsertBatchesSec.name, + Help: PerfCounterSQLBulkInsertBatchesSec.help, + }, + func() float64 { + return float64(atomic.LoadInt64(&PerfCounterSQLBulkInsertBatchesSec.value)) + }), + }, + { + &PerfCounterSQLDeletesSec, + prometheus.NewCounterFunc( + prometheus.CounterOpts{ + Namespace: PerfCounterSQLDeletesSec.nameSpace, + Subsystem: PerfCounterSQLDeletesSec.subSystem, + Name: PerfCounterSQLDeletesSec.name, + Help: PerfCounterSQLDeletesSec.help, + }, + func() float64 { + return float64(atomic.LoadInt64(&PerfCounterSQLDeletesSec.value)) + }), + }, + }, + } + + for _, w := range ctrs.counterValues { + prometheus.MustRegister(w.fun) + } + + return ctrs +} + +// list all the counters +// we can just read here without locking because if the counters get changed +// midway thru the loop, absent evidence to the contrary, the world will not end +func (p *PerformanceCounters) ListCounters() ([]PerformanceCounter, error) { + result := make([]PerformanceCounter, len(p.counterValues)) + for i, c := range p.counterValues { + result[i] = PerformanceCounter{ + NameSpace: c.ctr.nameSpace, + SubSystem: c.ctr.subSystem, + CounterName: c.ctr.name, + Value: c.ctr.value, + CounterType: c.ctr.counterType, + } + } + return result, nil +} diff --git a/prometheus/prometheus.go b/prometheus/prometheus.go deleted file mode 100644 index c0ff864a9..000000000 --- a/prometheus/prometheus.go +++ /dev/null @@ -1,308 +0,0 @@ -// Copyright 2022 Molecula Corp. (DBA FeatureBase). -// SPDX-License-Identifier: Apache-2.0 -package prometheus - -import ( - "sort" - "strings" - "sync" - "time" - - "github.com/featurebasedb/featurebase/v3/logger" - "github.com/featurebasedb/featurebase/v3/stats" - "github.com/prometheus/client_golang/prometheus" -) - -const ( - // namespace is prepended to each metric event name with "_" - defaultNamespace = "general" -) - -// Ensure client implements interface. -var _ stats.StatsClient = &prometheusClient{} - -// Module-level mutex to avoid copying in WithTags() -var mu sync.Mutex - -// prometheusClient represents a Prometheus implementation of pilosa.statsClient. -type prometheusClient struct { - tags []string - logger logger.Logger - counters map[string]prometheus.Counter - counterVecs map[string]*prometheus.CounterVec - gauges map[string]prometheus.Gauge - gaugeVecs map[string]*prometheus.GaugeVec - observers map[string]prometheus.Observer - summaryVecs map[string]*prometheus.SummaryVec - namespace string -} - -// ClientOption is a functional option type for prometheusClient -type ClientOption func(c *prometheusClient) - -// OptClientPrefix is a functional option on prometheusClient used to set the namespace -func OptClientNamespace(namespace string) ClientOption { - return func(c *prometheusClient) { - c.namespace = namespace - } -} - -// NewPrometheusClient returns a new instance of StatsClient. -func NewPrometheusClient(opts ...ClientOption) (*prometheusClient, error) { - client := &prometheusClient{ - logger: logger.NopLogger, - counters: make(map[string]prometheus.Counter), - counterVecs: make(map[string]*prometheus.CounterVec), - gauges: make(map[string]prometheus.Gauge), - gaugeVecs: make(map[string]*prometheus.GaugeVec), - observers: make(map[string]prometheus.Observer), - summaryVecs: make(map[string]*prometheus.SummaryVec), - namespace: defaultNamespace, - } - - for _, opt := range opts { - opt(client) - } - - return client, nil -} - -// Open no-op to satisfy interface -func (c *prometheusClient) Open() {} - -// Close no-op to satisfy interface -func (c *prometheusClient) Close() error { - return nil -} - -// Tags returns a sorted list of tags on the client. -func (c *prometheusClient) Tags() []string { - return c.tags -} - -// labels returns an instance of prometheus.Labels with the value of the set tags. -func (c *prometheusClient) labels() prometheus.Labels { - return tagsToLabels(c.tags, c.logger) -} - -// WithTags returns a new client with additional tags appended. -func (c *prometheusClient) WithTags(tags ...string) stats.StatsClient { - return &prometheusClient{ - tags: unionStringSlice(c.tags, tags), - logger: c.logger, - counters: c.counters, - counterVecs: c.counterVecs, - gauges: c.gauges, - gaugeVecs: c.gaugeVecs, - observers: c.observers, - summaryVecs: c.summaryVecs, - namespace: c.namespace, - } -} - -// Count tracks the number of times something occurs per second. -func (c *prometheusClient) Count(name string, value int64, rate float64) { - mu.Lock() - defer mu.Unlock() - - var counter prometheus.Counter - var ok bool - name = strings.Replace(name, ".", "_", -1) - labels := c.labels() - opts := prometheus.CounterOpts{ - Namespace: c.namespace, - Name: name, - } - if len(labels) == 0 { - counter, ok = c.counters[name] - if !ok { - counter = prometheus.NewCounter(opts) - c.counters[name] = counter - prometheus.MustRegister(counter) - } - } else { - var counterVec *prometheus.CounterVec - counterVec, ok = c.counterVecs[name] - if !ok { - counterVec = prometheus.NewCounterVec( - opts, - labelKeys(labels), - ) - c.counterVecs[name] = counterVec - prometheus.MustRegister(counterVec) - } - var err error - counter, err = counterVec.GetMetricWith(labels) - if err != nil { - c.logger.Errorf("counterVec.GetMetricWith error: %s", err) - } - } - if value == 1 { - counter.Inc() - } else { - counter.Add(float64(value)) - } -} - -// CountWithCustomTags tracks the number of times something occurs per second with custom tags. -func (c *prometheusClient) CountWithCustomTags(name string, value int64, rate float64, t []string) { - c.WithTags(append(c.tags, t...)...).Count(name, value, rate) -} - -// Gauge sets the value of a metric. -func (c *prometheusClient) Gauge(name string, value float64, rate float64) { - mu.Lock() - defer mu.Unlock() - - var gauge prometheus.Gauge - var ok bool - name = strings.Replace(name, ".", "_", -1) - labels := c.labels() - opts := prometheus.GaugeOpts{ - Namespace: c.namespace, - Name: name, - } - if len(labels) == 0 { - gauge, ok = c.gauges[name] - if !ok { - gauge = prometheus.NewGauge(opts) - c.gauges[name] = gauge - prometheus.MustRegister(gauge) - } - } else { - var gaugeVec *prometheus.GaugeVec - gaugeVec, ok = c.gaugeVecs[name] - if !ok { - gaugeVec = prometheus.NewGaugeVec( - opts, - labelKeys(labels), - ) - c.gaugeVecs[name] = gaugeVec - prometheus.MustRegister(gaugeVec) - } - var err error - gauge, err = gaugeVec.GetMetricWith(labels) - if err != nil { - c.logger.Errorf("gaugeVec.GetMetricWith error: %s", err) - return - } - } - gauge.Set(float64(value)) -} - -// Histogram tracks statistical distribution of a metric. -func (c *prometheusClient) Histogram(name string, value float64, rate float64) { - mu.Lock() - defer mu.Unlock() - - var observer prometheus.Observer - var ok bool - name = strings.Replace(name, ".", "_", -1) - labels := c.labels() - opts := prometheus.SummaryOpts{ - Namespace: c.namespace, - Name: name, - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - } - if len(labels) == 0 { - observer, ok = c.observers[name] - if !ok { - summary := prometheus.NewSummary(opts) - observer = summary - c.observers[name] = observer - prometheus.MustRegister(summary) - } - } else { - var summaryVec *prometheus.SummaryVec - summaryVec, ok = c.summaryVecs[name] - if !ok { - summaryVec = prometheus.NewSummaryVec( - opts, - labelKeys(labels), - ) - c.summaryVecs[name] = summaryVec - prometheus.MustRegister(summaryVec) - } - var err error - observer, err = summaryVec.GetMetricWith(labels) - if err != nil { - c.logger.Errorf("summaryVec.GetMetricWith error: %s", err) - return - } - } - observer.Observe(value) -} - -// Set tracks number of unique elements. -func (c *prometheusClient) Set(name string, value string, rate float64) { - c.logger.Infof("prometheusClient.Set unimplemented: %s=%s", name, value) -} - -// Timing tracks timing information for a metric. -func (c *prometheusClient) Timing(name string, value time.Duration, rate float64) { - c.Histogram(name, value.Seconds(), rate) -} - -// SetLogger sets the logger for client. -func (c *prometheusClient) SetLogger(logger logger.Logger) { - c.logger = logger -} - -// unionStringSlice returns a sorted set of tags which combine a & b. -func unionStringSlice(a, b []string) []string { - // Sort both sets first. - sort.Strings(a) - sort.Strings(b) - - // Find size of largest slice. - n := len(a) - if len(b) > n { - n = len(b) - } - - // Exit if both sets are empty. - if n == 0 { - return nil - } - - // Iterate over both in order and merge. - other := make([]string, 0, n) - for len(a) > 0 || len(b) > 0 { - if len(a) == 0 { - other, b = append(other, b[0]), b[1:] - } else if len(b) == 0 { - other, a = append(other, a[0]), a[1:] - } else if a[0] < b[0] { - other, a = append(other, a[0]), a[1:] - } else if b[0] < a[0] { - other, b = append(other, b[0]), b[1:] - } else { - other, a, b = append(other, a[0]), a[1:], b[1:] - } - } - return other -} - -func tagsToLabels(tags []string, logger logger.Logger) (labels prometheus.Labels) { - labels = make(prometheus.Labels) - for _, tag := range tags { - tagParts := strings.SplitAfterN(tag, ":", 2) - if len(tagParts) != 2 { - // only process tags in "key:value" form - logger.Errorf("invalid Prometheus label: %v\n", tag) - continue - } - labels[tagParts[0][0:len(tagParts[0])-1]] = tagParts[1] - } - return labels -} - -func labelKeys(labels prometheus.Labels) (keys []string) { - keys = make([]string, len(labels)) - i := 0 - for k := range labels { - keys[i] = k - i++ - } - return keys -} diff --git a/prometheus/prometheus_test.go b/prometheus/prometheus_test.go index 3eaca7842..391124ba8 100644 --- a/prometheus/prometheus_test.go +++ b/prometheus/prometheus_test.go @@ -3,62 +3,32 @@ package prometheus_test import ( - "reflect" "testing" - "time" pilosaPrometheus "github.com/featurebasedb/featurebase/v3/prometheus" "github.com/prometheus/client_golang/prometheus" io_prometheus_client "github.com/prometheus/client_model/go" ) -func TestPrometheusClient_WithTags(t *testing.T) { - // Create a new client. - c, err := pilosaPrometheus.NewPrometheusClient() - if err != nil { - t.Fatal(err) - } - defer c.Close() - - // Create a new client with additional tags. - c1 := c.WithTags("foo", "bar") - if tags := c1.Tags(); !reflect.DeepEqual(tags, []string{"bar", "foo"}) { - t.Fatalf("unexpected tags: %+v", tags) - } - - // Create a new client from the clone with more tags. - c2 := c1.WithTags("bar", "baz") - if tags := c2.Tags(); !reflect.DeepEqual(tags, []string{"bar", "baz", "foo"}) { - t.Fatalf("unexpected tags: %+v", tags) - } -} - func TestPrometheusClient_Methods(t *testing.T) { - // Create a new client. - c, err := pilosaPrometheus.NewPrometheusClient( - pilosaPrometheus.OptClientNamespace("testns"), - ) - if err != nil { - t.Fatal(err) - } + c := test.MustRunCluster(t, 1) defer c.Close() - dur, _ := time.ParseDuration("123us") - c.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"}) - c.Count("cc", 1, 1.0) - c.Gauge("gg", 10, 1.0) - c.Histogram("hh", 1, 1.0) - c.Timing("tt", dur, 1.0) - metricFams, err := prometheus.DefaultGatherer.Gather() if err != nil { t.Fatal(err) } - for _, metricName := range []string{"testns_ct", "testns_cc", "testns_gg", "testns_hh", "testns_tt"} { + for _, metricName := range []string{ + "pilosa_sql_statistics_sql_bulk_insert_batches_sec", + "pilosa_sql_statistics_sql_bulk_inserts_sec", + "pilosa_sql_statistics_sql_deletes_sec", + "pilosa_sql_statistics_sql_inserts_sec", + "pilosa_sql_statistics_sql_requests_sec", + } { if metricExists(metricName, metricFams) { continue } - t.Fatalf("Metric was not recorded: %s", metricName) + t.Fatalf("metric does not exist: %s", metricName) } } diff --git a/row.go b/row.go index 880f7c6a4..b525470e7 100644 --- a/row.go +++ b/row.go @@ -362,7 +362,7 @@ func (r *Row) Difference(others ...*Row) *Row { // be incorrect. // // Why unsupported? For a full description, see: -// https://github.com/molecula/pilosa/issues/403. +// https://github.com/featurebasedb/pilosa/issues/403. // In short, the current implementation will shift a bit // at the edge of a shard out of the shard and into a // container which is assumed to be an invalid container @@ -612,7 +612,7 @@ func (s *RowSegment) Xor(other *RowSegment) *RowSegment { // Shift returns s shifted by 1 bit. func (s *RowSegment) Shift() (*RowSegment, error) { // TODO: deal with overflow - // See issue: https://github.com/molecula/pilosa/issues/403 + // See issue: https://github.com/featurebasedb/pilosa/issues/403 data, err := s.data.Shift(1) if err != nil { return nil, errors.Wrap(err, "shifting roaring data") diff --git a/server.go b/server.go index 67cbad129..45efd6910 100644 --- a/server.go +++ b/server.go @@ -5,6 +5,7 @@ package pilosa import ( "context" "fmt" + "io" "log" "os" "os/exec" @@ -246,15 +247,6 @@ func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption { } } -// OptServerStatsClient is a functional option on Server -// used to specify the stats client. -func OptServerStatsClient(sc stats.StatsClient) ServerOption { - return func(s *Server) error { - s.holderConfig.StatsClient = sc - return nil - } -} - // OptServerDiagnosticsInterval is a functional option on Server // used to specify the duration between diagnostic checks. func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { @@ -542,7 +534,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { return nil, err } s.holder = NewHolder(path, s.holderConfig) - s.holder.Stats.SetLogger(s.logger) cwd, err := os.Getwd() if err != nil { return nil, err @@ -558,9 +549,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.cluster.sharder = s.sharder s.cluster.serverlessStorage = s.serverlessStorage - // Append the NodeID tag to stats. - s.holder.Stats = s.holder.Stats.WithTags(fmt.Sprintf("node_id:%s", s.nodeID)) - s.executor.Holder = s.holder s.holder.executor = s.executor s.executor.Cluster = s.cluster @@ -658,7 +646,6 @@ func (s *Server) Open() error { s.syncer.Node = node s.syncer.Cluster = s.cluster s.syncer.Closing = s.closing - s.syncer.Stats = s.holder.Stats.WithTags("component:HolderSyncer") // Start background process listening for translation // sync resets. @@ -1250,26 +1237,26 @@ func (s *Server) monitorRuntime() { return case <-s.gcNotifier.AfterGC(): // GC just ran. - s.holder.Stats.Count(MetricGarbageCollection, 1, 1.0) + CounterGarbageCollection.Inc() case <-ticker.C: } // Record the number of go routines. - s.holder.Stats.Gauge(MetricGoroutines, float64(runtime.NumGoroutine()), 1.0) + GaugeGoroutines.Set(float64(runtime.NumGoroutine())) openFiles, err := countOpenFiles() // Open File handles. if err == nil { - s.holder.Stats.Gauge(MetricOpenFiles, float64(openFiles), 1.0) + GaugeOpenFiles.Set(float64(openFiles)) } // Runtime memory metrics. runtime.ReadMemStats(&m) - s.holder.Stats.Gauge(MetricHeapAlloc, float64(m.HeapAlloc), 1.0) - s.holder.Stats.Gauge(MetricHeapInuse, float64(m.HeapInuse), 1.0) - s.holder.Stats.Gauge(MetricStackInuse, float64(m.StackInuse), 1.0) - s.holder.Stats.Gauge(MetricMallocs, float64(m.Mallocs), 1.0) - s.holder.Stats.Gauge(MetricFrees, float64(m.Frees), 1.0) + GaugeHeapAlloc.Set(float64(m.HeapAlloc)) + GaugeHeapInUse.Set(float64(m.HeapInuse)) + GaugeStackInUse.Set(float64(m.StackInuse)) + GaugeMallocs.Set(float64(m.Mallocs)) + GaugeFrees.Set(float64(m.Frees)) } } @@ -1413,6 +1400,10 @@ func (s *Server) CompileExecutionPlan(ctx context.Context, q string) (planner_ty return s.executionPlannerFn(s.executor, s.executor.client.api, q).CompilePlan(ctx, st) } +func (s *Server) RehydratePlanOperator(ctx context.Context, reader io.Reader) (planner_types.PlanOperator, error) { + return s.executionPlannerFn(s.executor, s.executor.client.api, "").RehydratePlanOp(ctx, reader) +} + // countOpenFiles on operating systems that support lsof. func countOpenFiles() (int, error) { switch runtime.GOOS { diff --git a/server/grpc.go b/server/grpc.go index c425da49b..cecfa3bea 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -41,12 +41,11 @@ type GRPCHandler struct { perms *authz.GroupPermissions logger logger.Logger queryLogger logger.Logger - stats stats.StatsClient inspectDeprecated sync.Once } func NewGRPCHandler(api *pilosa.API) *GRPCHandler { - return &GRPCHandler{api: api, logger: logger.NopLogger, stats: stats.NopStatsClient} + return &GRPCHandler{api: api, logger: logger.NopLogger} } func (h *GRPCHandler) WithLogger(logger logger.Logger) *GRPCHandler { @@ -54,11 +53,6 @@ func (h *GRPCHandler) WithLogger(logger logger.Logger) *GRPCHandler { return h } -func (h *GRPCHandler) WithStats(stats stats.StatsClient) *GRPCHandler { - h.stats = stats - return h -} - func (h *GRPCHandler) WithPerms(perms *authz.GroupPermissions) *GRPCHandler { h.perms = perms return h @@ -140,7 +134,7 @@ func errToStatusError(err error) error { } func (h *GRPCHandler) execSQL(ctx context.Context, queryStr string) (pb.ToRowser, error) { - h.stats.Count(pilosa.MetricSqlQueries, 1, 1) + pilosa.CounterSQLQueries.Inc() return execSQL(ctx, h.api, h.logger, queryStr) } @@ -340,9 +334,9 @@ func (h *GRPCHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQ return errToStatusError(err) } durFormat := time.Since(t) - h.stats.Timing(pilosa.MetricGRPCStreamQueryDurationSeconds, durQuery, 0.1) - h.stats.Timing(pilosa.MetricGRPCStreamFormatDurationSeconds, durFormat, 0.1) - h.stats.Count(pilosa.MetricPqlQueries, 1, 1) + pilosa.SummaryGRPCStreamQueryDurationSeconds.Observe(durQuery.Seconds()) + pilosa.SummaryGRPCStreamFormatDurationSeconds.Observe(durFormat.Seconds()) + pilosa.CounterPQLQueries.Inc() return errToStatusError(nil) } @@ -406,9 +400,9 @@ func (h *GRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest return nil, errors.Wrap(err, "sending header") } - h.stats.Timing(pilosa.MetricGRPCUnaryQueryDurationSeconds, durQuery, 0.1) - h.stats.Timing(pilosa.MetricGRPCUnaryFormatDurationSeconds, durFormat, 0.1) - h.stats.Count(pilosa.MetricPqlQueries, 1, 1) + pilosa.SummaryGRPCUnaryQueryDurationSeconds.Observe(durQuery.Seconds()) + pilosa.SummaryGRPCUnaryFormatDurationSeconds.Observe(durFormat.Seconds()) + pilosa.CounterPQLQueries.Inc() return table, errToStatusError(nil) } @@ -510,11 +504,10 @@ type VDSMGRPCHandler struct { grpcHandler *GRPCHandler api *pilosa.API logger logger.Logger - stats stats.StatsClient } func NewVDSMGRPCHandler(grpcHandler *GRPCHandler, api *pilosa.API) *VDSMGRPCHandler { - return &VDSMGRPCHandler{grpcHandler: grpcHandler, api: api, logger: logger.NopLogger, stats: stats.NopStatsClient} + return &VDSMGRPCHandler{grpcHandler: grpcHandler, api: api, logger: logger.NopLogger} } func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler { @@ -522,11 +515,6 @@ func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler { return h } -func (h *VDSMGRPCHandler) WithStats(stats stats.StatsClient) *VDSMGRPCHandler { - h.stats = stats - return h -} - // GetVDSs returns a single VDS given a name func (h *VDSMGRPCHandler) GetVDS(ctx context.Context, req *vdsm_pb.GetVDSRequest) (*vdsm_pb.GetVDSResponse, error) { typedIdOrName := req.GetIdOrName() @@ -1481,7 +1469,6 @@ type grpcServer struct { logger logger.Logger queryLogger logger.Logger - stats stats.StatsClient } type grpcServerOption func(s *grpcServer) error @@ -1514,13 +1501,6 @@ func OptGRPCServerLogger(logger logger.Logger) grpcServerOption { } } -func OptGRPCServerStats(stats stats.StatsClient) grpcServerOption { - return func(s *grpcServer) error { - s.stats = stats - return nil - } -} - func OptGRPCServerAuth(authn *authn.Auth) grpcServerOption { return func(s *grpcServer) error { s.auth = authn @@ -1640,7 +1620,7 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) { // create grpc server server.grpcServer = grpc.NewServer(gopts...) - grpcHandler := NewGRPCHandler(server.api).WithLogger(server.logger).WithStats(server.stats).WithQueryLogger(server.queryLogger) + grpcHandler := NewGRPCHandler(server.api).WithLogger(server.logger).WithQueryLogger(server.queryLogger) // add server permissions if we've got 'em if server.perms != nil { @@ -1648,7 +1628,7 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) { } pb.RegisterPilosaServer(server.grpcServer, grpcHandler) - vdsm_pb.RegisterMoleculaServer(server.grpcServer, NewVDSMGRPCHandler(grpcHandler, server.api).WithLogger(server.logger).WithStats(server.stats)) + vdsm_pb.RegisterMoleculaServer(server.grpcServer, NewVDSMGRPCHandler(grpcHandler, server.api).WithLogger(server.logger)) // register the server so its services are available to grpc_cli and others reflection.Register(server.grpcServer) diff --git a/server/handler_test.go b/server/handler_test.go index 765de0c81..fd6e48b37 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -1080,15 +1080,6 @@ func TestHandler_Endpoints(t *testing.T) { } }) - t.Run("Expvars", func(t *testing.T) { - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) - h.ServeHTTP(w, r) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - }) - t.Run("Recalculate Caches", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) diff --git a/server/server.go b/server/server.go index 3e9c22b26..72606c009 100644 --- a/server/server.go +++ b/server/server.go @@ -474,11 +474,6 @@ func (m *Command) setupServer() error { diagnosticsInterval = defaultDiagnosticsInterval } - statsClient, err := newStatsClient(m.Config.Metric.Service, m.Config.Metric.Host, m.Config.Namespace()) - if err != nil { - return errors.Wrap(err, "new stats client") - } - if m.Config.Listener == nil { m.ln, err = getListener(*uri, m.tlsConfig) if err != nil { @@ -581,7 +576,6 @@ func (m *Command) setupServer() error { pilosa.OptServerQueryLogger(m.queryLogger), pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()), pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()), - pilosa.OptServerStatsClient(statsClient), pilosa.OptServerURI(advertiseURI), pilosa.OptServerGRPCURI(advertiseGRPCURI), pilosa.OptServerClusterName(m.Config.Cluster.Name), @@ -691,7 +685,6 @@ func (m *Command) setupServer() error { OptGRPCServerListener(m.grpcLn), OptGRPCServerTLSConfig(m.tlsConfig), OptGRPCServerLogger(m.logger), - OptGRPCServerStats(statsClient), OptGRPCServerAuth(m.auth), OptGRPCServerPerm(&p), OptGRPCServerQueryLogger(m.queryLogger), @@ -828,24 +821,6 @@ func (m *Command) Close() error { } } -// newStatsClient creates a stats client from the config -func newStatsClient(name string, host string, namespace string) (stats.StatsClient, error) { - switch name { - case "expvar": - return stats.NewExpvarStatsClient(), nil - case "statsd": - return statsd.NewStatsClient(host, namespace) - case "prometheus": - return prometheus.NewPrometheusClient( - prometheus.OptClientNamespace(namespace), - ) - case "nop", "none": - return stats.NopStatsClient, nil - default: - return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, prometheus, none].", name) - } -} - // getListener gets a net.Listener based on the config. func getListener(uri pnet.URI, tlsconf *tls.Config) (ln net.Listener, err error) { // If bind URI has the https scheme, enable TLS diff --git a/sql3/interfaces.go b/sql3/interfaces.go index ee22cce56..9eddcc45d 100644 --- a/sql3/interfaces.go +++ b/sql3/interfaces.go @@ -3,6 +3,7 @@ package sql3 import ( "context" + "io" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" @@ -10,6 +11,7 @@ import ( type CompilePlanner interface { CompilePlan(context.Context, parser.Statement) (types.PlanOperator, error) + RehydratePlanOp(context.Context, io.Reader) (types.PlanOperator, error) } // Ensure type implements interface. @@ -25,3 +27,7 @@ func NewNopCompilePlanner() *NopCompilePlanner { func (p *NopCompilePlanner) CompilePlan(ctx context.Context, stmt parser.Statement) (types.PlanOperator, error) { return nil, nil } + +func (p *NopCompilePlanner) RehydratePlanOp(ctx context.Context, reader io.Reader) (types.PlanOperator, error) { + return nil, nil +} diff --git a/sql3/planner/compilecreatetable.go b/sql3/planner/compilecreatetable.go index 1d34aadae..2cbb77480 100644 --- a/sql3/planner/compilecreatetable.go +++ b/sql3/planner/compilecreatetable.go @@ -196,7 +196,13 @@ func (p *ExecutionPlanner) compileColumn(col *parser.ColumnDefinition) (*createT case dax.BaseTypeBool: column.fos = append(column.fos, pilosa.OptFieldTypeBool()) case dax.BaseTypeDecimal: - // Get the scale value. + + // if we don't have a scale, it's an error + if col.Type.Scale == nil { + return nil, sql3.NewErrDecimalScaleExpected(col.Type.Name.NamePos.Line, col.Type.Name.NamePos.Column) + } + + // get the scale value scale, err = strconv.ParseInt(col.Type.Scale.Value, 10, 64) if err != nil { return nil, err diff --git a/sql3/planner/compileselect.go b/sql3/planner/compileselect.go index abdca8222..02d5efdec 100644 --- a/sql3/planner/compileselect.go +++ b/sql3/planner/compileselect.go @@ -302,11 +302,16 @@ func (p *ExecutionPlanner) compileSource(scope *PlanOpQuery, source parser.Sourc // doing this check here because we don't have a 'system' flag that exists in the FB schema st, ok := systemTables[strings.ToLower(tableName)] if ok { + var op types.PlanOperator + op = NewPlanOpSystemTable(p, st) + if st.requiresFanout { + op = NewPlanOpFanout(p, op) + } if sourceExpr.Alias != nil { aliasName := parser.IdentName(sourceExpr.Alias) - return NewPlanOpRelAlias(aliasName, NewPlanOpSystemTable(p, st)), nil + return NewPlanOpRelAlias(aliasName, op), nil } - return NewPlanOpSystemTable(p, st), nil + return op, nil } // get all the columns for this table - we will eliminate unused ones diff --git a/sql3/planner/executionplanner.go b/sql3/planner/executionplanner.go index 5f7f59106..dbec61628 100644 --- a/sql3/planner/executionplanner.go +++ b/sql3/planner/executionplanner.go @@ -3,7 +3,12 @@ package planner import ( + "bytes" "context" + "fmt" + "io" + "net/http" + "strconv" pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/logger" @@ -79,6 +84,20 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen return rootOperator, err } +func (p *ExecutionPlanner) RehydratePlanOp(ctx context.Context, reader io.Reader) (types.PlanOperator, error) { + rdr := newWireProtocolParser(p, reader) + message, err := rdr.nextMessage() + if err != nil { + return nil, err + } + switch m := message.(type) { + case *messagePlanOp: + return m.op, nil + default: + return nil, sql3.NewErrInternalf("unexpected message type '%T'", message) + } +} + func (p *ExecutionPlanner) analyzePlan(stmt parser.Statement) error { switch stmt := stmt.(type) { case *parser.SelectStatement: @@ -120,3 +139,216 @@ const ( func (p *ExecutionPlanner) checkAccess(ctx context.Context, objectName string, _ accessType) error { return nil } + +type reduceFunc func(ctx context.Context, prev, v types.Rows) (types.Rows, error) + +type mapResponse struct { + node pilosa.ClusterNode + result types.Rows + err error +} + +func (e *ExecutionPlanner) mapReducePlanOp(ctx context.Context, op types.PlanOperator, reduceFn reduceFunc) (result types.Rows, err error) { + ch := make(chan mapResponse) + + // Wrap context with a cancel to kill goroutines on exit. + ctx, cancel := context.WithCancel(ctx) + // Create an errgroup so we can wait for all the goroutines to exit + eg, ctx := errgroup.WithContext(ctx) + + // After we're done processing, we have to wait for any outstanding + // functions in the ErrGroup to complete. If we didn't have an error + // already at that point, we'll report any errors from the ErrGroup + // instead. + defer func() { + cancel() + errWait := eg.Wait() + if err == nil { + err = errWait + } + }() + + nodes := e.systemAPI.ClusterNodes() + + // Start mapping across all nodes + if err = e.mapper(ctx, eg, ch, nodes, op, reduceFn); err != nil { + return nil, errors.Wrap(err, "starting mapper") + } + + // Iterate over all map responses and reduce. + expected := len(nodes) + done := ctx.Done() + for expected > 0 { + select { + case <-done: + return nil, ctx.Err() + case resp := <-ch: + if resp.err != nil { + return nil, errors.Wrap(resp.err, "query fanout") + } + // if we got a response that we aren't discarding + // because it's an error, subtract it from our count... + expected -= 1 + + // Reduce value. + + result, err = reduceFn(ctx, result, resp.result) + if err != nil { + cancel() + return nil, err + } + } + } + // note the deferred Wait above which might override this nil. + return result, nil +} + +func (e *ExecutionPlanner) mapper(ctx context.Context, eg *errgroup.Group, ch chan mapResponse, nodes []pilosa.ClusterNode, op types.PlanOperator, reduceFn reduceFunc) (reterr error) { + done := ctx.Done() + // Execute each node in a separate goroutine. + for _, node := range nodes { + node := node + eg.Go(func() error { + + resp := mapResponse{node: node} + + // Send local shards to mapper, otherwise remote exec. + if node.ID == e.systemAPI.NodeID() { + iter, err := op.Iterator(ctx, nil) + if err != nil { + resp.result = nil + resp.err = err + } + row, err := iter.Next(ctx) + if err != nil && err != types.ErrNoMoreRows { + resp.result = nil + resp.err = err + } + if err != types.ErrNoMoreRows { + for { + resp.result = append(resp.result, row) + row, err = iter.Next(ctx) + if err != nil && err != types.ErrNoMoreRows { + resp.result = nil + resp.err = err + } + if err == types.ErrNoMoreRows { + break + } + } + } + } else { + results, err := e.remotePlanExec(ctx, node.URI, op) + resp.result = results + resp.err = err + } + + // Return response to the channel. + select { + case <-done: + // If someone just canceled the context + // arbitrarily, we could end up here with this + // being the first non-nil error handed to + // the ErrGroup, in which case, it's the best + // explanation we have for why everything's + // stopping. + return ctx.Err() + case ch <- resp: + // If we return a non-nil error from this, the + // entire errGroup gets canceled. So we don't + // want to return a non-nil error if mapReduce + // might try to run another mapper against a + // different set of nodes. Note that this shouldn't + // matter; we just sent the error to mapReduce + // anyway, so it probably cancels the ErrGroup + // too. + if resp.err != nil { + return resp.err + } + } + return nil + }) + if reterr != nil { + return reterr // exit early if error occurs when running serially + } + } + return nil +} + +func (e *ExecutionPlanner) remotePlanExec(ctx context.Context, addr string, op types.PlanOperator) (types.Rows, error) { + b, err := writeOp(op) + if err != nil { + return nil, err + } + + // Create HTTP request. + u := fmt.Sprintf("%s/sql", addr) + req, err := http.NewRequest("POST", u, bytes.NewReader(b)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + // TODO (pok) internal auth + //AddAuthToken(ctx, &req.Header) + + req.Header.Set("Content-Length", strconv.Itoa(len(b))) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Accept", "application/octet-stream") + req.Header.Set("User-Agent", "pilosa/"+e.systemAPI.Version()) + req.Header.Set("X-FeatureBase-Plan-Operator", fmt.Sprintf("%T", op)) + + // Execute request against the host. + resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, sql3.NewErrInternalf("error posting internally: %s", err.Error()) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + // we have an error + return nil, sql3.NewErrInternalf("error posting internally: %d", resp.StatusCode) + } + + var rows types.Rows + parser := newWireProtocolParser(e, resp.Body) + state := 1 + for state <= 2 { + msg, err := parser.nextMessage() + if err != nil { + return nil, err + } + switch state { + case 1: + switch m := msg.(type) { + case *messageSchemaInfo: + parser.schema = m.schema + state = 2 + + case *messageError: + return nil, m.err + + default: + return nil, sql3.NewErrInternalf("unexpected token %d", msg.Token()) + } + + case 2: + switch m := msg.(type) { + case *messageRow: + rows = append(rows, m.row) + + case *messageDone: + // we're done + state = 3 + + case *messageError: + return nil, m.err + + default: + return nil, sql3.NewErrInternalf("unexpected token %d", msg.Token()) + } + + } + + } + return rows, nil +} diff --git a/sql3/planner/opbulkinsert.go b/sql3/planner/opbulkinsert.go index ca36b6b54..01f905c85 100644 --- a/sql3/planner/opbulkinsert.go +++ b/sql3/planner/opbulkinsert.go @@ -367,6 +367,8 @@ func (i *bulkInsertCSVRowIter) Next(ctx context.Context) (types.Row, error) { return nil, err } i.currentBatch = nil + // update the counter for bulk insert batches + pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1) } if i.options.rowsLimit > 0 && i.linesRead >= i.options.rowsLimit { break @@ -378,6 +380,8 @@ func (i *bulkInsertCSVRowIter) Next(ctx context.Context) (types.Row, error) { return nil, err } i.currentBatch = nil + // update the counter for bulk insert batches + pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1) } return nil, types.ErrNoMoreRows } @@ -777,6 +781,8 @@ func (i *bulkInsertNDJsonRowIter) Next(ctx context.Context) (types.Row, error) { return nil, err } i.currentBatch = nil + // update the counter for bulk insert batches + pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1) } if i.options.rowsLimit > 0 && i.linesRead >= i.options.rowsLimit { break @@ -788,6 +794,8 @@ func (i *bulkInsertNDJsonRowIter) Next(ctx context.Context) (types.Row, error) { return nil, err } i.currentBatch = nil + // update the counter for bulk insert batches + pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1) } return nil, types.ErrNoMoreRows } @@ -928,5 +936,9 @@ func processBatch(ctx context.Context, planner *ExecutionPlanner, tableName stri if err != nil && err != types.ErrNoMoreRows { return err } + + // update the counter for bulk inserts + pilosa.PerfCounterSQLBulkInsertsSec.Add(int64(len(insertValues))) + return nil } diff --git a/sql3/planner/opfanout.go b/sql3/planner/opfanout.go new file mode 100644 index 000000000..130e13c98 --- /dev/null +++ b/sql3/planner/opfanout.go @@ -0,0 +1,114 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "fmt" + + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" +) + +// PlanOpFanout is a query fanout operator that will execute an operator across all cluster nodes +type PlanOpFanout struct { + planner *ExecutionPlanner + ChildOp types.PlanOperator + warnings []string +} + +func NewPlanOpFanout(planner *ExecutionPlanner, child types.PlanOperator) *PlanOpFanout { + return &PlanOpFanout{ + planner: planner, + ChildOp: child, + warnings: make([]string, 0), + } +} + +func (p *PlanOpFanout) Schema() types.Schema { + return p.ChildOp.Schema() +} + +func (p *PlanOpFanout) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + return newFanOutIterator(p.planner, p.ChildOp), nil +} + +func (p *PlanOpFanout) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + if len(children) != 1 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return NewPlanOpFanout(p.planner, children[0]), nil +} + +func (p *PlanOpFanout) Children() []types.PlanOperator { + return []types.PlanOperator{ + p.ChildOp, + } +} + +func (p *PlanOpFanout) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + ps := make([]string, 0) + for _, e := range p.Schema() { + ps = append(ps, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeDescription())) + } + result["_schema"] = ps + result["child"] = p.ChildOp.Plan() + return result +} + +func (p *PlanOpFanout) String() string { + return "" +} + +func (p *PlanOpFanout) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpFanout) Warnings() []string { + return p.warnings +} + +func (p *PlanOpFanout) Expressions() []types.PlanExpression { + return []types.PlanExpression{} +} + +func (p *PlanOpFanout) WithUpdatedExpressions(exprs ...types.PlanExpression) (types.PlanOperator, error) { + if len(exprs) != 1 { + return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs)) + } + return NewPlanOpFilter(p.planner, exprs[0], p.ChildOp), nil +} + +type fanOutIterator struct { + planner *ExecutionPlanner + childOp types.PlanOperator + rows types.Rows +} + +func newFanOutIterator(planner *ExecutionPlanner, childOp types.PlanOperator) *fanOutIterator { + return &fanOutIterator{ + planner: planner, + childOp: childOp, + } +} + +func (i *fanOutIterator) Next(ctx context.Context) (types.Row, error) { + if i.rows == nil { + rows, err := i.planner.mapReducePlanOp(ctx, i.childOp, func(ctx context.Context, prev, v types.Rows) (types.Rows, error) { + return append(prev, v...), nil + }) + if err != nil { + return nil, err + } + i.rows = rows + } + if len(i.rows) > 0 { + row := i.rows[0] + // Move to next result element. + i.rows = i.rows[1:] + return row, nil + } + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/opinsert.go b/sql3/planner/opinsert.go index 33c20981d..626c1dbce 100644 --- a/sql3/planner/opinsert.go +++ b/sql3/planner/opinsert.go @@ -368,5 +368,8 @@ func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) { return nil, errors.Wrap(err, "importing batch") } + // update the counter for inserts + pilosa.PerfCounterSQLInsertsSec.Add(int64(batch.Len())) + return nil, types.ErrNoMoreRows } diff --git a/sql3/planner/opsystemtable.go b/sql3/planner/opsystemtable.go index 949b63bfe..23a810ec4 100644 --- a/sql3/planner/opsystemtable.go +++ b/sql3/planner/opsystemtable.go @@ -17,16 +17,18 @@ import ( // exclude this file from SonarCloud dupe eval const ( - fbClusterInfo = "fb_cluster_info" - fbClusterNodes = "fb_cluster_nodes" - fbExecRequests = "fb_exec_requests" + fbClusterInfo = "fb_cluster_info" + fbClusterNodes = "fb_cluster_nodes" + fbExecRequests = "fb_exec_requests" + fbPerformanceCounters = "fb_performance_counters" fbTableDDL = "fb_table_ddl" ) type systemTable struct { - name string - schema types.Schema + name string + schema types.Schema + requiresFanout bool } var systemTables = map[string]*systemTable{ @@ -74,6 +76,7 @@ var systemTables = map[string]*systemTable{ Type: parser.NewDataTypeInt(), }, }, + requiresFanout: false, }, fbClusterNodes: { name: fbClusterNodes, @@ -109,11 +112,17 @@ var systemTables = map[string]*systemTable{ Type: parser.NewDataTypeBool(), }, }, + requiresFanout: false, }, fbExecRequests: { name: fbExecRequests, schema: types.Schema{ + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "nodeid", + Type: parser.NewDataTypeString(), + }, &types.PlannerColumn{ RelationName: fbExecRequests, ColumnName: "request_id", @@ -195,6 +204,7 @@ var systemTables = map[string]*systemTable{ Type: parser.NewDataTypeString(), }, }, + requiresFanout: true, }, fbTableDDL: { @@ -216,6 +226,44 @@ var systemTables = map[string]*systemTable{ Type: parser.NewDataTypeString(), }, }, + requiresFanout: false, + }, + + fbPerformanceCounters: { + name: fbPerformanceCounters, + schema: types.Schema{ + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "nodeid", + Type: parser.NewDataTypeString(), + }, + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "namespace", + Type: parser.NewDataTypeString(), + }, + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "subsystem", + Type: parser.NewDataTypeString(), + }, + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "counter_name", + Type: parser.NewDataTypeString(), + }, + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "value", + Type: parser.NewDataTypeInt(), + }, + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "counter_type", + Type: parser.NewDataTypeInt(), + }, + }, + requiresFanout: true, }, } @@ -279,6 +327,10 @@ func (p *PlanOpSystemTable) Iterator(ctx context.Context, row types.Row) (types. return &fbTableDDLRowIter{ planner: p.planner, }, nil + case fbPerformanceCounters: + return &fbPerformanceCountersRowIter{ + planner: p.planner, + }, nil default: return nil, sql3.NewErrInternalf("unable to find system table '%s'", p.table.name) } @@ -367,9 +419,11 @@ func (i *fbExecRequestsRowIter) Next(ctx context.Context) (types.Row, error) { } } + nodeId := i.planner.systemAPI.NodeID() if len(i.result) > 0 { n := i.result[0] row := []interface{}{ + nodeId, n.RequestID, n.UserID, n.StartTime, @@ -533,3 +587,37 @@ func (i *fbTableDDLRowIter) Next(ctx context.Context) (types.Row, error) { } return nil, types.ErrNoMoreRows } + +type fbPerformanceCountersRowIter struct { + planner *ExecutionPlanner + result []pilosa.PerformanceCounter +} + +var _ types.RowIterator = (*fbPerformanceCountersRowIter)(nil) + +func (i *fbPerformanceCountersRowIter) Next(ctx context.Context) (types.Row, error) { + if i.result == nil { + var err error + i.result, err = pilosa.PerfCounters.ListCounters() + if err != nil { + return nil, err + } + } + + nodeId := i.planner.systemAPI.NodeID() + if len(i.result) > 0 { + n := i.result[0] + row := []interface{}{ + nodeId, + n.NameSpace, + n.SubSystem, + n.CounterName, + n.Value, + n.CounterType, + } + // Move to next result element. + i.result = i.result[1:] + return row, nil + } + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/types/operator.go b/sql3/planner/types/operator.go index ef36f1650..a0e843c03 100644 --- a/sql3/planner/types/operator.go +++ b/sql3/planner/types/operator.go @@ -80,9 +80,12 @@ func (r Schema) Plan() []map[string]interface{} { return result } -// Row is a tuple of values +// Row is a tuple (of values) type Row []interface{} +// Rows is a table of rows +type Rows []Row + // Append appends all the values in r2 to this row and returns the result func (r Row) Append(r2 Row) Row { row := make(Row, len(r)+len(r2)) diff --git a/sql3/planner/wireprotocol.go b/sql3/planner/wireprotocol.go new file mode 100644 index 000000000..b43a90fb2 --- /dev/null +++ b/sql3/planner/wireprotocol.go @@ -0,0 +1,211 @@ +package planner + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "io" + + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/wireprotocol" +) + +type wireProtocolMessage interface { + Token() int16 +} + +func writeOp(op types.PlanOperator) ([]byte, error) { + buf := new(bytes.Buffer) + writer := bufio.NewWriter(buf) + // serialize a plan op - for now we are just supporting + // system tables, and we'll send the name of the system table + switch op := op.(type) { + case *PlanOpSystemTable: + // write token + b := make([]byte, 2) + binary.BigEndian.PutUint16(b, uint16(wireprotocol.TOKEN_PLAN_OP)) + writer.Write(b) + // write table name len + t := op.table.name + b = make([]byte, 4) + binary.BigEndian.PutUint32(b, uint32(len(t))) + writer.Write(b) + // write table name + writer.WriteString(op.table.name) + writer.Flush() + default: + return []byte{}, sql3.NewErrInternalf("unexpected plan operator type '%T'", op) + } + return buf.Bytes(), nil +} + +type wireProtocolParser struct { + planner *ExecutionPlanner + reader io.Reader + schema types.Schema +} + +func newWireProtocolParser(p *ExecutionPlanner, reader io.Reader) *wireProtocolParser { + return &wireProtocolParser{ + planner: p, + reader: reader, + } +} + +func (f *wireProtocolParser) nextMessage() (wireProtocolMessage, error) { + var t int16 + err := binary.Read(f.reader, binary.BigEndian, &t) + if err != nil { + return nil, err + } + switch t { + case wireprotocol.TOKEN_SCHEMA_INFO: + return newMessageSchemaInfo(f.planner, f.reader) + + case wireprotocol.TOKEN_ROW: + if f.schema == nil { + return nil, sql3.NewErrInternalf("schema uninitialized") + } + return newMessageRow(f.planner, f.reader, f.schema) + + case wireprotocol.TOKEN_ERROR_MESSAGE: + return newMessageError(f.planner, f.reader) + + case wireprotocol.TOKEN_DONE: + return newMessageDone(f.planner, f.reader) + + case wireprotocol.TOKEN_PLAN_OP: + return newMessagePlanOp(f.planner, f.reader) + + default: + return nil, sql3.NewErrInternalf("unexpected token %d", t) + } +} + +type messagePlanOp struct { + token int16 + op types.PlanOperator +} + +var _ wireProtocolMessage = (*messagePlanOp)(nil) + +func newMessagePlanOp(p *ExecutionPlanner, reader io.Reader) (*messagePlanOp, error) { + + var len int32 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + bname := make([]byte, len) + err = binary.Read(reader, binary.BigEndian, &bname) + if err != nil { + return nil, err + } + name := string(bname) + + st, ok := systemTables[name] + if !ok { + return nil, sql3.NewErrInternalf("unexpected system table name %s", name) + } + + return &messagePlanOp{ + token: wireprotocol.TOKEN_PLAN_OP, + op: NewPlanOpSystemTable(p, st), + }, nil +} + +func (m *messagePlanOp) Token() int16 { + return m.token +} + +type messageError struct { + token int16 + err error +} + +var _ wireProtocolMessage = (*messageError)(nil) + +func newMessageError(p *ExecutionPlanner, reader io.Reader) (*messageError, error) { + var len int32 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + bname := make([]byte, len) + err = binary.Read(reader, binary.BigEndian, &bname) + if err != nil { + return nil, err + } + errMsg := string(bname) + + return &messageError{ + token: wireprotocol.TOKEN_ERROR_MESSAGE, + err: errors.New(errMsg), + }, nil +} + +func (m *messageError) Token() int16 { + return m.token +} + +type messageSchemaInfo struct { + token int16 + schema types.Schema +} + +var _ wireProtocolMessage = (*messageSchemaInfo)(nil) + +func newMessageSchemaInfo(p *ExecutionPlanner, reader io.Reader) (*messageSchemaInfo, error) { + schema, err := wireprotocol.ReadSchema(reader) + if err != nil { + return nil, err + } + return &messageSchemaInfo{ + token: wireprotocol.TOKEN_SCHEMA_INFO, + schema: schema, + }, nil +} + +func (m *messageSchemaInfo) Token() int16 { + return m.token +} + +type messageRow struct { + token int16 + row types.Row +} + +var _ wireProtocolMessage = (*messageRow)(nil) + +func newMessageRow(p *ExecutionPlanner, reader io.Reader, schema types.Schema) (*messageRow, error) { + row, err := wireprotocol.ReadRow(reader, schema) + if err != nil { + return nil, err + } + return &messageRow{ + token: wireprotocol.TOKEN_ROW, + row: row, + }, nil +} + +func (m *messageRow) Token() int16 { + return m.token +} + +type messageDone struct { + token int16 +} + +var _ wireProtocolMessage = (*messageDone)(nil) + +func newMessageDone(p *ExecutionPlanner, reader io.Reader) (*messageDone, error) { + return &messageDone{ + token: wireprotocol.TOKEN_DONE, + }, nil +} + +func (m *messageDone) Token() int16 { + return m.token +} diff --git a/sql3/sql_complex_test.go b/sql3/sql_complex_test.go index 38bd3dac4..55c0a26c3 100644 --- a/sql3/sql_complex_test.go +++ b/sql3/sql_complex_test.go @@ -36,6 +36,92 @@ func TestPlanner_Misc(t *testing.T) { assert.True(t, d.EqualTo(pql.NewDecimal(12345678, 6))) } +func TestPlanner_SystemTableFanout(t *testing.T) { + c := test.MustRunCluster(t, 3) + defer c.Close() + + server := c.GetNode(0).Server + + t.Run("PerfCounters", func(t *testing.T) { + results, columns, err := sql_test.MustQueryRows(t, server, `select * from fb_performance_counters`) + if err != nil { + t.Fatal(err) + } + if len(results) != 15 { + t.Fatal(fmt.Errorf("unexpected result set length")) + } + + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldString("nodeid"), + wireQueryFieldString("namespace"), + wireQueryFieldString("subsystem"), + wireQueryFieldString("counter_name"), + wireQueryFieldInt("value"), + wireQueryFieldInt("counter_type"), + }, columns); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("SystemTablesExecRequests", func(t *testing.T) { + results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select * from fb_exec_requests`) + if err != nil { + t.Fatal(err) + } + + if len(results) != 2 { + t.Fatal(fmt.Errorf("unexpected result set length")) + } + + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldString("nodeid"), + wireQueryFieldString("request_id"), + wireQueryFieldString("user"), + wireQueryFieldTimestamp("start_time"), + wireQueryFieldTimestamp("end_time"), + wireQueryFieldString("status"), + wireQueryFieldString("wait_type"), + wireQueryFieldInt("wait_time"), + wireQueryFieldString("wait_resource"), + wireQueryFieldInt("cpu_time"), + wireQueryFieldInt("elapsed_time"), + wireQueryFieldInt("reads"), + wireQueryFieldInt("writes"), + wireQueryFieldInt("logical_reads"), + wireQueryFieldInt("row_count"), + wireQueryFieldString("sql"), + wireQueryFieldString("plan"), + }, columns); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("SystemTablesExecRequestsAgg", func(t *testing.T) { + _, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select + count(request_id) as request_count, + min(elapsed_time) as min_duration, + max(elapsed_time) as max_duration, + avg(elapsed_time) as avg_duration + from + fb_exec_requests + where + status = 'complete';`) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("request_count"), + wireQueryFieldInt("min_duration"), + wireQueryFieldInt("max_duration"), + wireQueryFieldDecimal("avg_duration", 4), + }, columns); diff != "" { + t.Fatal(diff) + } + }) + +} + func TestPlanner_Show(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -102,64 +188,12 @@ func TestPlanner_Show(t *testing.T) { } }) - t.Run("SystemTablesExecRequests", func(t *testing.T) { - _, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select * from fb_exec_requests`) - if err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff([]*pilosa.WireQueryField{ - wireQueryFieldString("request_id"), - wireQueryFieldString("user"), - wireQueryFieldTimestamp("start_time"), - wireQueryFieldTimestamp("end_time"), - wireQueryFieldString("status"), - wireQueryFieldString("wait_type"), - wireQueryFieldInt("wait_time"), - wireQueryFieldString("wait_resource"), - wireQueryFieldInt("cpu_time"), - wireQueryFieldInt("elapsed_time"), - wireQueryFieldInt("reads"), - wireQueryFieldInt("writes"), - wireQueryFieldInt("logical_reads"), - wireQueryFieldInt("row_count"), - wireQueryFieldString("sql"), - wireQueryFieldString("plan"), - }, columns); diff != "" { - t.Fatal(diff) - } - }) - - t.Run("SystemTablesExecRequestsAgg", func(t *testing.T) { - _, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select - count(request_id) as request_count, - min(elapsed_time) as min_duration, - max(elapsed_time) as max_duration, - avg(elapsed_time) as avg_duration - from - fb_exec_requests - where - status = 'complete';`) - if err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff([]*pilosa.WireQueryField{ - wireQueryFieldInt("request_count"), - wireQueryFieldInt("min_duration"), - wireQueryFieldInt("max_duration"), - wireQueryFieldDecimal("avg_duration", 4), - }, columns); diff != "" { - t.Fatal(diff) - } - }) - t.Run("ShowTables", func(t *testing.T) { results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SHOW TABLES`) if err != nil { t.Fatal(err) } - if len(results) != 6 { + if len(results) != 7 { t.Fatal(fmt.Errorf("unexpected result set length")) } diff --git a/stats/stats.go b/stats/stats.go deleted file mode 100644 index d85b900fd..000000000 --- a/stats/stats.go +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2022 Molecula Corp. (DBA FeatureBase). -// SPDX-License-Identifier: Apache-2.0 -package stats - -import ( - "expvar" - "sort" - "strings" - "sync" - "time" - - "github.com/featurebasedb/featurebase/v3/logger" -) - -// Expvar global expvar map. -var Expvar *expvar.Map - -// StatsClient represents a client to a stats server. -type StatsClient interface { - // Returns a sorted list of tags on the client. - Tags() []string - - // Returns a new client with additional tags appended. - WithTags(tags ...string) StatsClient - - // Tracks the number of times something occurs per second. - Count(name string, value int64, rate float64) - - // Tracks the number of times something occurs per second with custom tags - CountWithCustomTags(name string, value int64, rate float64, tags []string) - - // Sets the value of a metric. - Gauge(name string, value float64, rate float64) - - // Tracks statistical distribution of a metric. - Histogram(name string, value float64, rate float64) - - // Tracks number of unique elements. - Set(name string, value string, rate float64) - - // Tracks timing information for a metric. - Timing(name string, value time.Duration, rate float64) - - // SetLogger Set the logger output type - SetLogger(logger logger.Logger) - - // Starts the service - Open() - - // Closes the client - Close() error -} - -// NopStatsClient represents a client that doesn't do anything. -var NopStatsClient StatsClient = &nopStatsClient{} - -type nopStatsClient struct{} - -func (c *nopStatsClient) Tags() []string { return nil } -func (c *nopStatsClient) WithTags(tags ...string) StatsClient { return c } -func (c *nopStatsClient) Count(name string, value int64, rate float64) {} -func (c *nopStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) {} -func (c *nopStatsClient) Gauge(name string, value float64, rate float64) {} -func (c *nopStatsClient) Histogram(name string, value float64, rate float64) {} -func (c *nopStatsClient) Set(name string, value string, rate float64) {} -func (c *nopStatsClient) Timing(name string, value time.Duration, rate float64) {} -func (c *nopStatsClient) SetLogger(logger logger.Logger) {} -func (c *nopStatsClient) Open() {} -func (c *nopStatsClient) Close() error { return nil } - -// expvarStatsClient writes stats out to expvars. -type expvarStatsClient struct { - mu sync.Mutex - m *expvar.Map - tags []string -} - -// NewExpvarStatsClient returns a new instance of ExpvarStatsClient. -// This client points at the root of the expvar index map. -func NewExpvarStatsClient() *expvarStatsClient { - if Expvar == nil { - Expvar = expvar.NewMap("index") - } - return &expvarStatsClient{ - m: Expvar, - } -} - -// Tags returns a sorted list of tags on the client. -func (c *expvarStatsClient) Tags() []string { - return nil -} - -// WithTags returns a new client with additional tags appended. -func (c *expvarStatsClient) WithTags(tags ...string) StatsClient { - m := &expvar.Map{} - m.Init() - c.m.Set(strings.Join(tags, ","), m) - - return &expvarStatsClient{ - m: m, - tags: unionStringSlice(c.tags, tags), - } -} - -// Count tracks the number of times something occurs. -func (c *expvarStatsClient) Count(name string, value int64, rate float64) { - c.m.Add(name, value) -} - -// CountWithCustomTags Tracks the number of times something occurs per second with custom tags -func (c *expvarStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) { - c.m.Add(name, value) -} - -// Gauge sets the value of a metric. -func (c *expvarStatsClient) Gauge(name string, value float64, rate float64) { - var f expvar.Float - f.Set(value) - c.m.Set(name, &f) -} - -// Histogram tracks statistical distribution of a metric. -// This works the same as gauge for this client. -func (c *expvarStatsClient) Histogram(name string, value float64, rate float64) { - c.Gauge(name, value, rate) -} - -// Set tracks number of unique elements. -func (c *expvarStatsClient) Set(name string, value string, rate float64) { - var s expvar.String - s.Set(value) - c.m.Set(name, &s) -} - -// Timing tracks timing information for a metric. -func (c *expvarStatsClient) Timing(name string, value time.Duration, rate float64) { - c.mu.Lock() - d, _ := c.m.Get(name).(time.Duration) - c.m.Set(name, d+value) - c.mu.Unlock() -} - -// SetLogger has no logger. -func (c *expvarStatsClient) SetLogger(logger logger.Logger) { -} - -// Open no-op. -func (c *expvarStatsClient) Open() {} - -// Close no-op. -func (c *expvarStatsClient) Close() error { return nil } - -// MultiStatsClient joins multiple stats clients together. -type MultiStatsClient []StatsClient - -// Tags returns tags from the first client. -func (a MultiStatsClient) Tags() []string { - if len(a) > 0 { - return a[0].Tags() - } - return nil -} - -// WithTags returns a new set of clients with the additional tags. -func (a MultiStatsClient) WithTags(tags ...string) StatsClient { - other := make(MultiStatsClient, len(a)) - for i := range a { - other[i] = a[i].WithTags(tags...) - } - return other -} - -// Count tracks the number of times something occurs per second on all clients. -func (a MultiStatsClient) Count(name string, value int64, rate float64) { - for _, c := range a { - c.Count(name, value, rate) - } -} - -// CountWithCustomTags Tracks the number of times something occurs per second with custom tags -func (a MultiStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) { - for _, c := range a { - c.CountWithCustomTags(name, value, rate, tags) - } -} - -// Gauge sets the value of a metric on all clients. -func (a MultiStatsClient) Gauge(name string, value float64, rate float64) { - for _, c := range a { - c.Gauge(name, value, rate) - } -} - -// Histogram tracks statistical distribution of a metric on all clients. -func (a MultiStatsClient) Histogram(name string, value float64, rate float64) { - for _, c := range a { - c.Histogram(name, value, rate) - } -} - -// Set tracks number of unique elements on all clients. -func (a MultiStatsClient) Set(name string, value string, rate float64) { - for _, c := range a { - c.Set(name, value, rate) - } -} - -// Timing tracks timing information for a metric on all clients. -func (a MultiStatsClient) Timing(name string, value time.Duration, rate float64) { - for _, c := range a { - c.Timing(name, value, rate) - } -} - -// SetLogger Sets the StatsD logger output type. -func (a MultiStatsClient) SetLogger(logger logger.Logger) { - for _, c := range a { - c.SetLogger(logger) - } -} - -// Open starts the stat service. -func (a MultiStatsClient) Open() { - for _, c := range a { - c.Open() - } -} - -// Close shuts down the stats clients. -func (a MultiStatsClient) Close() error { - for _, c := range a { - err := c.Close() - if err != nil { - return err - } - } - return nil -} - -// unionStringSlice returns a sorted set of tags which combine a & b. -func unionStringSlice(a, b []string) []string { - // Sort both sets first. - sort.Strings(a) - sort.Strings(b) - - // Find size of largest slice. - n := len(a) - if len(b) > n { - n = len(b) - } - - // Exit if both sets are empty. - if n == 0 { - return nil - } - - // Iterate over both in order and merge. - other := make([]string, 0, n) - for len(a) > 0 || len(b) > 0 { - if len(a) == 0 { - other, b = append(other, b[0]), b[1:] - } else if len(b) == 0 { - other, a = append(other, a[0]), a[1:] - } else if a[0] < b[0] { - other, a = append(other, a[0]), a[1:] - } else if b[0] < a[0] { - other, b = append(other, b[0]), b[1:] - } else { - other, a, b = append(other, a[0]), a[1:], b[1:] - } - } - return other -} diff --git a/stats/stats_test.go b/stats/stats_test.go deleted file mode 100644 index ae0824be6..000000000 --- a/stats/stats_test.go +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright 2022 Molecula Corp. (DBA FeatureBase). -// SPDX-License-Identifier: Apache-2.0 -package stats_test - -import ( - "context" - "fmt" - "net/http/httptest" - "strings" - "testing" - "time" - - pilosa "github.com/featurebasedb/featurebase/v3" - "github.com/featurebasedb/featurebase/v3/logger" - "github.com/featurebasedb/featurebase/v3/stats" - "github.com/featurebasedb/featurebase/v3/test" -) - -// TestMultiStatClient_Expvar run the multistat client with exp var -// since the EXPVAR data is stored in a global we should run these in one test function -func TestMultiStatClient_Expvar(t *testing.T) { - hldr := test.MustOpenHolder(t) - - c := stats.NewExpvarStatsClient() - ms := make(stats.MultiStatsClient, 1) - ms[0] = c - hldr.Stats = ms - - hldr.SetBit("d", "f", 0, 0) - hldr.SetBit("d", "f", 0, 1) - hldr.SetBit("d", "f", 0, pilosa.ShardWidth) - hldr.SetBit("d", "f", 0, pilosa.ShardWidth+2) - hldr.ClearBit("d", "f", 0, 1) - - indexStats := fmt.Sprintf(`{"%s": %d, "%s": %d}`, pilosa.MetricClearBit, 1, pilosa.MetricSetBit, 4) - - if stats.Expvar.String() != `{"index:d": `+indexStats+`}` { - t.Fatalf("unexpected expvar : %s", stats.Expvar.String()) - } - - hldr.Stats.CountWithCustomTags("cc", 1, 1.0, []string{"foo:bar"}) - if stats.Expvar.String() != `{"cc": 1, "index:d": `+indexStats+`}` { - t.Fatalf("unexpected expvar : %s", stats.Expvar.String()) - } - - // Gauge creates a unique key, subsequent Gauge calls will overwrite - hldr.Stats.Gauge("g", 5, 1.0) - hldr.Stats.Gauge("g", 8, 1.0) - if stats.Expvar.String() != `{"cc": 1, "g": 8, "index:d": `+indexStats+`}` { - t.Fatalf("unexpected expvar : %s", stats.Expvar.String()) - } - - // Set creates a unique key, subsequent sets will overwrite - hldr.Stats.Set("s", "4", 1.0) - hldr.Stats.Set("s", "7", 1.0) - if stats.Expvar.String() != `{"cc": 1, "g": 8, "index:d": `+indexStats+`, "s": "7"}` { - t.Fatalf("unexpected expvar : %s", stats.Expvar.String()) - } - - // Record timing duration and a uniquely Set key/value - dur, _ := time.ParseDuration("123us") - hldr.Stats.Timing("tt", dur, 1.0) - if stats.Expvar.String() != `{"cc": 1, "g": 8, "index:d": `+indexStats+`, "s": "7", "tt": 123µs}` { - t.Fatalf("unexpected expvar : %s", stats.Expvar.String()) - } - - // Expvar histogram is implemented as a gauge - hldr.Stats.Histogram("hh", 3, 1.0) - if stats.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": `+indexStats+`, "s": "7", "tt": 123µs}` { - t.Fatalf("unexpected expvar : %s", stats.Expvar.String()) - } - - // Expvar should ignore earlier set tags from setbit - if hldr.Stats.Tags() != nil { - t.Fatalf("unexpected tag") - } -} - -func TestStatsCount_TopN(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()} - - // Execute query. - called := false - hldr.Holder.Stats = &MockStats{ - mockCountWithTags: func(name string, value int64, rate float64, tags []string) { - if name != "query_topn_total" { - t.Errorf("Expected query_topn_total, Results %s", name) - } - - if tags[0] != "index:d" { - t.Errorf("Expected index, Results %s", tags[0]) - } - - called = true - }, - } - - hldr.SetBit("d", "f", 0, 0) - hldr.SetBit("d", "f", 0, 1) - hldr.SetBit("d", "f", 0, pilosa.ShardWidth) - hldr.SetBit("d", "f", 0, pilosa.ShardWidth+2) - - if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `TopN(field=f, n=2)`}); err != nil { - t.Fatal(err) - } - if !called { - t.Error("CountWithCustomTags name isn't called") - } -} - -func TestStatsCount_Bitmap(t *testing.T) { - // Cluster has to be unhsared because we're mocking the stats which writes - // to a holder in use by other tests. - c := test.MustRunUnsharedCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()} - called := false - hldr.Holder.Stats = &MockStats{ - mockCountWithTags: func(name string, value int64, rate float64, tags []string) { - if name != pilosa.MetricRow { - t.Errorf("Expected %s, Results %s", pilosa.MetricRow, name) - } - - if tags[0] != "index:d" { - t.Errorf("Expected index, Results %s", tags[0]) - } - - called = true - }, - } - - hldr.SetBit("d", "f", 0, 0) - hldr.SetBit("d", "f", 0, 1) - - if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `Row(f=0)`}); err != nil { - t.Fatal(err) - } - if !called { - t.Error("CountWithCustomTags name isn't called") - } -} - -func TestStatsCount_APICalls(t *testing.T) { - // We can't share a cluster when we're modifying its stats counter. - cluster := test.MustRunUnsharedCluster(t, 1) - defer cluster.Close() - cmd := cluster.GetNode(0) - h := cmd.Handler.(*pilosa.Handler).Handler - holder := cmd.Server.Holder() - hldr := test.Holder{Holder: holder} - - t.Run("create index", func(t *testing.T) { - called := false - hldr.Stats = &MockStats{ - mockCount: func(name string, value int64, rate float64) { - if name != pilosa.MetricCreateIndex { - t.Errorf("Expected %v, Results %s", pilosa.MetricCreateIndex, name) - } - called = true - }, - } - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i", strings.NewReader(""))) - if !called { - t.Error("Count isn't called") - } - }) - - t.Run("create field", func(t *testing.T) { - called := false - hldr.Stats = &MockStats{ - mockCountWithTags: func(name string, value int64, rate float64, index []string) { - if name != pilosa.MetricCreateField { - t.Errorf("Expected %v, Results %s", pilosa.MetricCreateField, name) - } - if index[0] != "index:i" { - t.Errorf("Expected index:i, Results %s", index) - } - - called = true - }, - } - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/field/f", strings.NewReader(""))) - if !called { - t.Error("Count isn't called") - } - }) - - t.Run("delete field", func(t *testing.T) { - called := false - hldr.Stats = &MockStats{ - mockCountWithTags: func(name string, value int64, rate float64, index []string) { - if name != pilosa.MetricDeleteField { - t.Errorf("Expected %v, Results %s", pilosa.MetricDeleteField, name) - } - if index[0] != "index:i" { - t.Errorf("Expected index:i, Results %s", index) - } - - called = true - }, - } - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f", strings.NewReader(""))) - if !called { - t.Error("Count isn't called") - } - }) - - t.Run("delete index", func(t *testing.T) { - called := false - hldr.Stats = &MockStats{ - mockCount: func(name string, value int64, rate float64) { - if name != pilosa.MetricDeleteIndex { - t.Errorf("Expected %v, Results %s", pilosa.MetricDeleteIndex, name) - } - - called = true - }, - } - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader(""))) - if !called { - t.Error("Count isn't called") - } - }) - -} - -type MockStats struct { - mockCount func(name string, value int64, rate float64) - mockCountWithTags func(name string, value int64, rate float64, tags []string) -} - -func (s *MockStats) Count(name string, value int64, rate float64) { - if s.mockCount != nil { - s.mockCount(name, value, rate) - } -} - -func (s *MockStats) CountWithCustomTags(name string, value int64, rate float64, tags []string) { - if s.mockCountWithTags != nil { - s.mockCountWithTags(name, value, rate, tags) - } -} - -func (c *MockStats) Tags() []string { return nil } -func (c *MockStats) WithTags(tags ...string) stats.StatsClient { return c } -func (c *MockStats) Gauge(name string, value float64, rate float64) {} -func (c *MockStats) Histogram(name string, value float64, rate float64) {} -func (c *MockStats) Set(name string, value string, rate float64) {} -func (c *MockStats) Timing(name string, value time.Duration, rate float64) {} -func (c *MockStats) SetLogger(logger logger.Logger) {} -func (c *MockStats) Open() {} -func (c *MockStats) Close() error { return nil } diff --git a/statsd/statsd.go b/statsd/statsd.go deleted file mode 100644 index 8a6900ab3..000000000 --- a/statsd/statsd.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2022 Molecula Corp. (DBA FeatureBase). -// SPDX-License-Identifier: Apache-2.0 -package statsd - -import ( - "sort" - "time" - - "github.com/DataDog/datadog-go/statsd" - "github.com/featurebasedb/featurebase/v3/logger" - "github.com/featurebasedb/featurebase/v3/stats" -) - -// StatsD protocol wrapper using the DataDog library that added Tags to the StatsD protocol -// statsD defailt host is "127.0.0.1:8125" - -const ( - // bufferLen Stats lient buffer size. - bufferLen = 1024 -) - -// Ensure client implements interface. -var _ stats.StatsClient = &statsClient{} - -// statsClient represents a StatsD implementation of pilosa.statsClient. -type statsClient struct { - client *statsd.Client - tags []string - logger logger.Logger - - // prefix is appended to each metric event name - prefix string -} - -// NewStatsClient returns a new instance of StatsClient. -func NewStatsClient(host string, namespace string) (*statsClient, error) { - c, err := statsd.NewBuffered(host, bufferLen) - if err != nil { - return nil, err - } - - return &statsClient{ - client: c, - logger: logger.NopLogger, - prefix: namespace + ".", - }, nil -} - -// Open no-op -func (c *statsClient) Open() {} - -// Close closes the connection to the agent. -func (c *statsClient) Close() error { - return c.client.Close() -} - -// Tags returns a sorted list of tags on the client. -func (c *statsClient) Tags() []string { - return c.tags -} - -// WithTags returns a new client with additional tags appended. -func (c *statsClient) WithTags(tags ...string) stats.StatsClient { - return &statsClient{ - client: c.client, - tags: unionStringSlice(c.tags, tags), - logger: c.logger, - } -} - -// Count tracks the number of times something occurs per second. -func (c *statsClient) Count(name string, value int64, rate float64) { - if err := c.client.Count(c.prefix+name, value, c.tags, rate); err != nil { - c.logger.Errorf("statsd.StatsClient.Count error: %s", err) - } -} - -// CountWithCustomTags tracks the number of times something occurs per second with custom tags. -func (c *statsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) { - tags := append(c.tags, t...) - if err := c.client.Count(c.prefix+name, value, tags, rate); err != nil { - c.logger.Errorf("statsd.StatsClient.Count error: %s", err) - } -} - -// Gauge sets the value of a metric. -func (c *statsClient) Gauge(name string, value float64, rate float64) { - if err := c.client.Gauge(c.prefix+name, value, c.tags, rate); err != nil { - c.logger.Errorf("statsd.StatsClient.Gauge error: %s", err) - } -} - -// Histogram tracks statistical distribution of a metric. -func (c *statsClient) Histogram(name string, value float64, rate float64) { - if err := c.client.Histogram(c.prefix+name, value, c.tags, rate); err != nil { - c.logger.Errorf("statsd.StatsClient.Histogram error: %s", err) - } -} - -// Set tracks number of unique elements. -func (c *statsClient) Set(name string, value string, rate float64) { - if err := c.client.Set(c.prefix+name, value, c.tags, rate); err != nil { - c.logger.Errorf("statsd.StatsClient.Set error: %s", err) - } -} - -// Timing tracks timing information for a metric. -func (c *statsClient) Timing(name string, value time.Duration, rate float64) { - if err := c.client.Timing(c.prefix+name, value, c.tags, rate); err != nil { - c.logger.Errorf("statsd.StatsClient.Timing error: %s", err) - } -} - -// SetLogger sets the logger for client. -func (c *statsClient) SetLogger(logger logger.Logger) { - c.logger = logger -} - -// unionStringSlice returns a sorted set of tags which combine a & b. -func unionStringSlice(a, b []string) []string { - // Sort both sets first. - sort.Strings(a) - sort.Strings(b) - - // Find size of largest slice. - n := len(a) - if len(b) > n { - n = len(b) - } - - // Exit if both sets are empty. - if n == 0 { - return nil - } - - // Iterate over both in order and merge. - other := make([]string, 0, n) - for len(a) > 0 || len(b) > 0 { - if len(a) == 0 { - other, b = append(other, b[0]), b[1:] - } else if len(b) == 0 { - other, a = append(other, a[0]), a[1:] - } else if a[0] < b[0] { - other, a = append(other, a[0]), a[1:] - } else if b[0] < a[0] { - other, b = append(other, b[0]), b[1:] - } else { - other, a, b = append(other, a[0]), a[1:], b[1:] - } - } - return other -} diff --git a/statsd/statsd_test.go b/statsd/statsd_test.go deleted file mode 100644 index dcb855205..000000000 --- a/statsd/statsd_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2022 Molecula Corp. (DBA FeatureBase). -// SPDX-License-Identifier: Apache-2.0 -package statsd_test - -import ( - "reflect" - "testing" - "time" - - "github.com/featurebasedb/featurebase/v3/statsd" - _ "github.com/featurebasedb/featurebase/v3/test" -) - -func TestStatsClient_WithTags(t *testing.T) { - // Create a new client. - c, err := statsd.NewStatsClient("localhost:19444", "testnamespace") - if err != nil { - t.Fatal(err) - } - defer c.Close() - - // Create a new client with additional tags. - c1 := c.WithTags("foo", "bar") - if tags := c1.Tags(); !reflect.DeepEqual(tags, []string{"bar", "foo"}) { - t.Fatalf("unexpected tags: %+v", tags) - } - - // Create a new client from the clone with more tags. - c2 := c1.WithTags("bar", "baz") - if tags := c2.Tags(); !reflect.DeepEqual(tags, []string{"bar", "baz", "foo"}) { - t.Fatalf("unexpected tags: %+v", tags) - } -} - -func TestStatsClient_Methods(t *testing.T) { - // Create a new client. - c, err := statsd.NewStatsClient("localhost:19444", "testnamespace") - if err != nil { - t.Fatal(err) - } - defer c.Close() - - dur, _ := time.ParseDuration("123us") - c.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"}) - c.Count("cc", 1, 1.0) - c.Gauge("gg", 10, 1.0) - c.Histogram("hh", 1, 1.0) - c.Timing("tt", dur, 1.0) - c.Set("ss", "ss", 1.0) -} diff --git a/view.go b/view.go index bba4927c2..8be79bb17 100644 --- a/view.go +++ b/view.go @@ -51,7 +51,6 @@ type view struct { fragments map[uint64]*fragment broadcaster broadcaster - stats stats.StatsClient knownShards *roaring.Bitmap knownShardsCopied uint32 @@ -79,7 +78,6 @@ func newView(holder *Holder, path, index, field, name string, fieldOptions Field fragments: make(map[uint64]*fragment), broadcaster: NopBroadcaster, - stats: stats.NopStatsClient, knownShards: roaring.NewSliceBitmap(), closing: make(chan struct{}), @@ -393,7 +391,6 @@ func (v *view) newFragment(shard uint64) *fragment { frag := newFragment(v.holder, v.idx, v.fld, v, shard) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize - frag.stats = v.stats if v.fieldType == FieldTypeMutex { frag.mutexVector = newRowsVector(frag) } else if v.fieldType == FieldTypeBool { diff --git a/wireprotocol/wireprimitives.go b/wireprotocol/wireprimitives.go new file mode 100644 index 000000000..be88ebd1d --- /dev/null +++ b/wireprotocol/wireprimitives.go @@ -0,0 +1,563 @@ +package wireprotocol + +import ( + "bufio" + "bytes" + "encoding/binary" + "time" + + "io" + + "github.com/featurebasedb/featurebase/v3/errors" + "github.com/featurebasedb/featurebase/v3/pql" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" +) + +const ( + // server --> client + TOKEN_SCHEMA_INFO int16 = 0xA1 + TOKEN_ROW int16 = 0xA2 + TOKEN_DONE int16 = 0xFD + TOKEN_INFO_MESSAGE int16 = 0xFE + TOKEN_ERROR_MESSAGE int16 = 0xFF + + // client --> server + TOKEN_SQL int16 = 0x01 + TOKEN_PLAN_OP int16 = 0x02 +) + +const ( + TYPE_VOID int8 = 0x00 + TYPE_ID int8 = 0x01 + TYPE_BOOL int8 = 0x02 + TYPE_INT int8 = 0x03 + TYPE_DECIMAL int8 = 0x04 + TYPE_TIMESTAMP int8 = 0x05 + TYPE_IDSET int8 = 0x06 + TYPE_STRING int8 = 0x07 + TYPE_STRINGSET int8 = 0x08 +) + +func ExpectToken(reader io.Reader, token int16) (int16, error) { + var tk int16 + err := binary.Read(reader, binary.BigEndian, &tk) + if err != nil { + return 0, err + } + if tk != token { + return 0, errors.Errorf("expected token found %d", token) + } + return tk, nil +} + +// TOKEN_COLUMN_INFO message +// length (bytes) +// token 2 +// column count 2 +// +// (n) columns +// +// name length 1 +// name (from prev) +// data type 1 +// +// (optional) +// if type decimal +// scale 1 + +// note RelationName and AliasName members from +// PlannerColumn are not sent over the wire +func WriteSchema(schema types.Schema) ([]byte, error) { + buf := new(bytes.Buffer) + writer := bufio.NewWriter(buf) + // write token + writeToken(writer, TOKEN_SCHEMA_INFO) + + // column count + writeInt16(writer, int16(len(schema))) + + // for each column + for _, s := range schema { + // name len byte + writeInt8(writer, int8(len(s.ColumnName))) + // name + writer.WriteString(s.ColumnName) + // type byte + switch ty := s.Type.(type) { + case *parser.DataTypeID: + writeInt8(writer, TYPE_ID) + + case *parser.DataTypeBool: + writeInt8(writer, TYPE_BOOL) + + case *parser.DataTypeInt: + writeInt8(writer, TYPE_INT) + + case *parser.DataTypeDecimal: + writeInt8(writer, TYPE_DECIMAL) + writeInt8(writer, int8(ty.Scale)) + + case *parser.DataTypeTimestamp: + writeInt8(writer, TYPE_TIMESTAMP) + + case *parser.DataTypeIDSet: + writeInt8(writer, TYPE_IDSET) + + case *parser.DataTypeString: + writeInt8(writer, TYPE_STRING) + + case *parser.DataTypeStringSet: + writeInt8(writer, TYPE_STRINGSET) + + default: + return []byte{}, errors.Errorf("unexpected type '%T'", s.Type) + } + } + writer.Flush() + return buf.Bytes(), nil +} + +func ReadSchema(reader io.Reader) (types.Schema, error) { + var columnCount int16 + err := binary.Read(reader, binary.BigEndian, &columnCount) + if err != nil { + return nil, err + } + + var schema types.Schema + for i := 0; i < int(columnCount); i++ { + + var nameLen int8 + err = binary.Read(reader, binary.BigEndian, &nameLen) + if err != nil { + return nil, err + } + bname := make([]byte, nameLen) + err = binary.Read(reader, binary.BigEndian, &bname) + if err != nil { + return nil, err + } + colName := string(bname) + + var typ int8 + err = binary.Read(reader, binary.BigEndian, &typ) + if err != nil { + return nil, err + } + var dataType parser.ExprDataType + + switch typ { + case TYPE_ID: + dataType = parser.NewDataTypeID() + + case TYPE_BOOL: + dataType = parser.NewDataTypeBool() + + case TYPE_INT: + dataType = parser.NewDataTypeInt() + + case TYPE_DECIMAL: + var scale int8 + err = binary.Read(reader, binary.BigEndian, &scale) + if err != nil { + return nil, err + } + dataType = parser.NewDataTypeDecimal(int64(scale)) + + case TYPE_TIMESTAMP: + dataType = parser.NewDataTypeTimestamp() + + case TYPE_IDSET: + dataType = parser.NewDataTypeIDSet() + + case TYPE_STRING: + dataType = parser.NewDataTypeString() + + case TYPE_STRINGSET: + dataType = parser.NewDataTypeStringSet() + } + + schema = append(schema, &types.PlannerColumn{ + ColumnName: colName, + Type: dataType, + }) + } + return schema, nil +} + +// TOKEN_ROW message +// length (bytes) +// token 2 +// +// (n) columns +// if column length is 0 --> null +// +// - for ID, INT +// +// column length 1 +// value 8 +// +// - for DECIMAL +// +// column length 1 +// value 8 +// +// - for BOOL +// +// column length 1 +// value 1 +// +// - for TIMESTAMP +// +// column length 1 +// value 8 +// +// - for IDSET +// +// set length 2 +// (n) values +// value 8 +// +// - for STRING +// +// column length 2 +// value (from prev) +// +// - for STRINGSET +// +// set length 2 +// (n) values +// value len 2 +// value (from prev) +// + +func WriteRow(row types.Row, schema types.Schema) ([]byte, error) { + buf := new(bytes.Buffer) + writer := bufio.NewWriter(buf) + // write token + writeToken(writer, TOKEN_ROW) + + // for each column + for i, s := range schema { + val := row[i] + switch s.Type.(type) { + case *parser.DataTypeID, *parser.DataTypeInt: + if val == nil { + writeInt8(writer, 0) + } else { + writeInt8(writer, 8) + v, ok := row[i].(int64) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt64(writer, v) + } + + case *parser.DataTypeDecimal: + if val == nil { + writeInt8(writer, 0) + } else { + writeInt8(writer, 8) + v, ok := row[i].(pql.Decimal) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt64(writer, v.ToInt64(v.Scale)) + } + + case *parser.DataTypeBool: + if val == nil { + writeInt8(writer, 0) + } else { + writeInt8(writer, 1) + v, ok := row[i].(bool) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + if v { + writeInt8(writer, 1) + } else { + writeInt8(writer, 0) + } + } + + case *parser.DataTypeTimestamp: + if val == nil { + writeInt8(writer, 0) + } else { + writeInt8(writer, 8) + v, ok := row[i].(time.Time) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt64(writer, v.UnixNano()) + } + + case *parser.DataTypeIDSet: + if val == nil { + writeInt16(writer, 0) + } else { + v, ok := row[i].([]int64) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt16(writer, int16(len(v))) + for _, s := range v { + writeInt64(writer, s) + } + } + + case *parser.DataTypeString: + if val == nil { + writeInt16(writer, 0) + } else { + v, ok := row[i].(string) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt16(writer, int16(len(v))) + writer.WriteString(v) + } + + case *parser.DataTypeStringSet: + if val == nil { + writeInt16(writer, 0) + } else { + v, ok := row[i].([]string) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt16(writer, int16(len(v))) + for _, s := range v { + writeInt16(writer, int16(len(s))) + writer.WriteString(s) + } + } + + default: + return []byte{}, errors.Errorf("unexpected type '%T'", s.Type) + } + } + writer.Flush() + return buf.Bytes(), nil +} + +func ReadRow(reader io.Reader, schema types.Schema) (types.Row, error) { + + row := make(types.Row, len(schema)) + + for idx, s := range schema { + switch t := s.Type.(type) { + case *parser.DataTypeID, *parser.DataTypeInt: + var len int8 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + var value int64 + err := binary.Read(reader, binary.BigEndian, &value) + if err != nil { + return nil, err + } + row[idx] = value + } + + case *parser.DataTypeDecimal: + var len int8 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + var value int64 + err := binary.Read(reader, binary.BigEndian, &value) + if err != nil { + return nil, err + } + row[idx] = pql.NewDecimal(value, t.Scale) + } + + case *parser.DataTypeBool: + var len int8 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + var value int8 + err := binary.Read(reader, binary.BigEndian, &value) + if err != nil { + return nil, err + } + row[idx] = value == 1 + } + + case *parser.DataTypeTimestamp: + var len int8 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + var value int64 + err := binary.Read(reader, binary.BigEndian, &value) + if err != nil { + return nil, err + } + row[idx] = time.Unix(0, value) + } + + case *parser.DataTypeIDSet: + var len int16 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + set := make([]int64, len) + for j, _ := range set { + var value int64 + err := binary.Read(reader, binary.BigEndian, &value) + if err != nil { + return nil, err + } + set[j] = value + } + row[idx] = set + } + + case *parser.DataTypeString: + var len int16 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + bvalue := make([]byte, len) + err = binary.Read(reader, binary.BigEndian, &bvalue) + if err != nil { + return nil, err + } + row[idx] = string(bvalue) + } + + case *parser.DataTypeStringSet: + var len int16 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + set := make([]string, len) + for j, _ := range set { + var vlen int16 + err = binary.Read(reader, binary.BigEndian, &vlen) + if err != nil { + return nil, err + } + bvalue := make([]byte, vlen) + err = binary.Read(reader, binary.BigEndian, &bvalue) + if err != nil { + return nil, err + } + set[j] = string(bvalue) + } + row[idx] = set + } + + default: + return nil, errors.Errorf("unexpected type '%T'", s.Type) + } + } + + return row, nil +} + +// TOKEN_DONE message +// length (bytes) +// token 2 + +func WriteDone() []byte { + buf := new(bytes.Buffer) + writer := bufio.NewWriter(buf) + // write token + writeToken(writer, TOKEN_DONE) + writer.Flush() + return buf.Bytes() +} + +// TOKEN_ERROR_MESSAGE message +// length (bytes) +// token 2 +// +// message len 4 +// message (from prev) + +func WriteError(err error) []byte { + buf := new(bytes.Buffer) + writer := bufio.NewWriter(buf) + // write token + writeToken(writer, TOKEN_ERROR_MESSAGE) + // write error len + t := err.Error() + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, uint32(len(t))) + writer.Write(b) + // write error + writer.WriteString(t) + writer.Flush() + + return buf.Bytes() +} + +func ReadError(reader io.Reader) (string, error) { + var len int16 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return "", err + } + bvalue := make([]byte, len) + err = binary.Read(reader, binary.BigEndian, &bvalue) + if err != nil { + return "", err + } + return string(bvalue), nil +} + +func writeToken(w io.Writer, token int16) { + writeInt16(w, token) +} + +func writeInt8(w io.Writer, i int8) { + b := make([]byte, 1) + b[0] = byte(i) + w.Write(b) +} + +func writeInt16(w io.Writer, i int16) { + b := make([]byte, 2) + binary.BigEndian.PutUint16(b, uint16(i)) + w.Write(b) +} + +func writeInt64(w io.Writer, i int64) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(i)) + w.Write(b) +} diff --git a/wireprotocol/wireprimitives_test.go b/wireprotocol/wireprimitives_test.go new file mode 100644 index 000000000..79f7a2fe2 --- /dev/null +++ b/wireprotocol/wireprimitives_test.go @@ -0,0 +1,144 @@ +package wireprotocol_test + +import ( + "bytes" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/featurebasedb/featurebase/v3/pql" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/wireprotocol" +) + +func TestWireProtocol_Schema(t *testing.T) { + + s := types.Schema{ + &types.PlannerColumn{ + ColumnName: "col1", + Type: parser.NewDataTypeID(), + }, + &types.PlannerColumn{ + ColumnName: "col2", + Type: parser.NewDataTypeInt(), + }, + &types.PlannerColumn{ + ColumnName: "col3", + Type: parser.NewDataTypeDecimal(4), + }, + &types.PlannerColumn{ + ColumnName: "col4", + Type: parser.NewDataTypeString(), + }, + &types.PlannerColumn{ + ColumnName: "col5", + Type: parser.NewDataTypeStringSet(), + }, + &types.PlannerColumn{ + ColumnName: "col6", + Type: parser.NewDataTypeIDSet(), + }, + &types.PlannerColumn{ + ColumnName: "col7", + Type: parser.NewDataTypeBool(), + }, + &types.PlannerColumn{ + ColumnName: "col8", + Type: parser.NewDataTypeTimestamp(), + }, + } + + b, err := wireprotocol.WriteSchema(s) + if err != nil { + t.Fatal(err) + } + + rdr := bytes.NewReader(b) + _, err = wireprotocol.ExpectToken(rdr, wireprotocol.TOKEN_SCHEMA_INFO) + if err != nil { + t.Fatal(err) + } + + sr, err := wireprotocol.ReadSchema(rdr) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(s, sr); diff != "" { + t.Fatal(diff) + } +} + +func TestWireProtocol_Row(t *testing.T) { + + s := types.Schema{ + &types.PlannerColumn{ + ColumnName: "col1", + Type: parser.NewDataTypeID(), + }, + &types.PlannerColumn{ + ColumnName: "col2", + Type: parser.NewDataTypeInt(), + }, + &types.PlannerColumn{ + ColumnName: "col3", + Type: parser.NewDataTypeDecimal(4), + }, + &types.PlannerColumn{ + ColumnName: "col4", + Type: parser.NewDataTypeString(), + }, + &types.PlannerColumn{ + ColumnName: "col5", + Type: parser.NewDataTypeStringSet(), + }, + &types.PlannerColumn{ + ColumnName: "col6", + Type: parser.NewDataTypeIDSet(), + }, + &types.PlannerColumn{ + ColumnName: "col7", + Type: parser.NewDataTypeBool(), + }, + &types.PlannerColumn{ + ColumnName: "col8", + Type: parser.NewDataTypeTimestamp(), + }, + } + + r := types.Row{ + int64(1), + int64(2), + pql.NewDecimal(123400, 4), + string("foo"), + []string{"bar", "baz"}, + []int64{10, 20}, + bool(false), + time.Now().UTC(), + } + + b, err := wireprotocol.WriteRow(r, s) + if err != nil { + t.Fatal(err) + } + + rdr := bytes.NewReader(b) + _, err = wireprotocol.ExpectToken(rdr, wireprotocol.TOKEN_ROW) + if err != nil { + t.Fatal(err) + } + + rr, err := wireprotocol.ReadRow(rdr, s) + if err != nil { + t.Fatal(err) + } + + opt := cmp.Comparer(func(x, y pql.Decimal) bool { + return x.EqualTo(y) + }) + + if diff := cmp.Diff(r, rr, opt); diff != "" { + t.Fatal(diff) + } +} From adbad90fe10042f1eb2a3e40385021d02e9f12c7 Mon Sep 17 00:00:00 2001 From: tgruben Date: Thu, 12 Jan 2023 12:58:19 -0600 Subject: [PATCH 02/15] string support for dataframe (#2405) * String support for arrow (cherry picked from commit 4cd3cb02a2cdb7fc864113d043342c6d12aa51c9) --- apply.go | 117 +++++++++++++++++++++++++++++-- arrow.go | 10 ++- ctl/dataframe-csv-loader.go | 14 +++- ctl/dataframe-csv-loader_test.go | 27 +++++++ dataframe_test.go | 45 +++++++++++- go.mod | 2 +- go.sum | 4 +- http_handler.go | 1 + 8 files changed, 207 insertions(+), 13 deletions(-) diff --git a/apply.go b/apply.go index 12f89aa25..127bbe8f6 100644 --- a/apply.go +++ b/apply.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "sort" "strings" "sync" @@ -260,14 +261,14 @@ func (e *executor) executeApplyShard(ctx context.Context, qcx *Qcx, index string func NewShardFile(ctx context.Context, name string, mem memory.Allocator, e *executor) (*ShardFile, error) { if !e.dataFrameExists(name) { - return &ShardFile{dest: name, executor: e}, nil + return &ShardFile{dest: name, executor: e, strings: make(map[key][]string)}, nil } // else read in existing table, err := e.getDataTable(ctx, name, mem) if err != nil { return nil, err } - return &ShardFile{table: table, schema: table.Schema(), dest: name, executor: e}, nil + return &ShardFile{table: table, schema: table.Schema(), dest: name, executor: e, strings: make(map[key][]string)}, nil } type NameType struct { @@ -291,6 +292,8 @@ func cast(v interface{}) arrow.DataType { return arrow.PrimitiveTypes.Float64 case float64: return arrow.PrimitiveTypes.Float64 + case *arrow.StringType: + return arrow.BinaryTypes.String default: vprint.VV("%T .... %v", v, v) } @@ -305,6 +308,11 @@ func (cr *ChangesetRequest) ArrowSchema() *arrow.Schema { return arrow.NewSchema(fields, nil) } +type key struct { + col int + chunk int +} + type ShardFile struct { table arrow.Table schema *arrow.Schema @@ -313,6 +321,7 @@ type ShardFile struct { columns []interface{} dest string executor *executor + strings map[key][]string } func compareSchema(s1, s2 *arrow.Schema) bool { @@ -365,6 +374,8 @@ func (sf *ShardFile) buildAppenders(maxid int64) { sf.columns[i] = make([]int64, newSize) case arrow.PrimitiveTypes.Float64: sf.columns[i] = make([]float64, newSize) + case arrow.BinaryTypes.String: + sf.columns[i] = make([]string, newSize) } } sf.added = newSize @@ -381,6 +392,11 @@ func (sf *ShardFile) SetFloatValue(col int, row int64, val float64) { v[row-sf.beforeRows] = val } +func (sf *ShardFile) SetStringValue(col int, row int64, val string) { + v := sf.columns[col].([]string) + v[row-sf.beforeRows] = val +} + func (sf *ShardFile) Process(cs *ChangesetRequest) error { err := sf.process(cs) if err != nil { @@ -394,9 +410,34 @@ func (sf *ShardFile) Process(cs *ChangesetRequest) error { return os.Rename(rtemp+sf.executor.TableExtension(), sf.dest+sf.executor.TableExtension()) } +func (sf *ShardFile) LoadBlobs() error { + for col := 0; col < len(sf.schema.Fields()); col++ { + column := sf.table.Column(col) + switch column.DataType() { + case arrow.BinaryTypes.String: + for i, chunk := range column.Data().Chunks() { + stringData := chunk.(*array.String) + k := key{col: col, chunk: i} + for j := 0; j < stringData.Len(); j++ { + v := stringData.Value(j) + sf.strings[k] = append(sf.strings[k], v) + + } + } + } + } + return nil +} + +func (sf *ShardFile) ReplaceString(col, chunk, l int, s string) { + sf.strings[key{col: col, chunk: chunk}][l] = s +} + func (sf *ShardFile) process(cs *ChangesetRequest) error { offset := 0 if sf.table != nil { + // need to load blobs prior + sf.LoadBlobs() column := sf.table.Column(0) resolver := dataframe.NewChunkResolver(column) for i, rowid := range cs.ShardIds { @@ -414,6 +455,10 @@ func (sf *ShardFile) process(cs *ChangesetRequest) error { case arrow.PrimitiveTypes.Float64: v := column.Data().Chunk(chunk).(*array.Float64).Float64Values() v[l] = cs.Columns[col].([]float64)[i] + case arrow.BinaryTypes.String: + // TODO(twg) 2023/01/09 How to update existing? + new := cs.Columns[col].([]string)[i] + sf.ReplaceString(col, chunk, l, new) default: panic(fmt.Sprintf("Unknown Type %v", column.DataType())) } @@ -433,6 +478,8 @@ func (sf *ShardFile) process(cs *ChangesetRequest) error { sf.SetIntValue(col, rowid, cs.Columns[col].([]int64)[i]) case arrow.PrimitiveTypes.Float64: sf.SetFloatValue(col, rowid, cs.Columns[col].([]float64)[i]) + case arrow.BinaryTypes.String: + sf.SetStringValue(col, rowid, cs.Columns[col].([]string)[i]) default: panic(fmt.Sprintf("2 Unknown Type %v", sf.schema.Field(col).Type)) } @@ -443,15 +490,65 @@ func (sf *ShardFile) process(cs *ChangesetRequest) error { return nil } +type twoSlices struct { + id_slice []int + lists_slice [][]string +} + +type SortByOther twoSlices + +func (sbo SortByOther) Len() int { + return len(sbo.id_slice) +} + +func (sbo SortByOther) Swap(i, j int) { + sbo.id_slice[i], sbo.id_slice[j] = sbo.id_slice[j], sbo.id_slice[i] + sbo.lists_slice[i], sbo.lists_slice[j] = sbo.lists_slice[j], sbo.lists_slice[i] +} + +func (sbo SortByOther) Less(i, j int) bool { + return sbo.id_slice[i] < sbo.id_slice[j] +} + +func (sf *ShardFile) buildFromStrings(idx int, mem memory.Allocator) []arrow.Array { + ids := make([]int, 0) + lists := make([][]string, 0) + for k, v := range sf.strings { + if k.col == idx { // ugh not ordered :( + ids = append(ids, k.chunk) + lists = append(lists, v) + } + } + // sort ids/lists + parts := twoSlices{id_slice: ids, lists_slice: lists} + sort.Sort(SortByOther(parts)) + + builder := array.NewStringBuilder(mem) + chunks := make([]arrow.Array, 0) + for _, v := range parts.lists_slice { + builder.AppendValues(v, nil) + newChunk := builder.NewArray() + chunks = append(chunks, newChunk) + } + return chunks +} + func (sf *ShardFile) Save(name string) error { parts := make([]arrow.Array, 0) mem := memory.NewGoAllocator() for col := 0; col < len(sf.schema.Fields()); col++ { chunks := make([]arrow.Array, 0) if sf.table != nil { - // we append if there was existing parquet file + // we append if there was existing file column := sf.table.Column(col) - chunks = append(chunks, column.Data().Chunks()...) + // if primative type + switch column.DataType() { + case arrow.BinaryTypes.String: + chunks = sf.buildFromStrings(col, mem) + default: + chunks = append(chunks, column.Data().Chunks()...) + } + // else binary type } switch sf.schema.Field(col).Type { case arrow.PrimitiveTypes.Int64: @@ -480,6 +577,18 @@ func (sf *ShardFile) Save(name string) error { return err } parts = append(parts, record) + case arrow.BinaryTypes.String: + if sf.added > 0 { + fbuild := array.NewStringBuilder(mem) + fbuild.AppendValues(sf.columns[col].([]string), nil) // TODO(twg) 2022/09/28 need to handle null + newChunk := fbuild.NewArray() + chunks = append(chunks, newChunk) + } + record, err := array.Concatenate(chunks, mem) + if err != nil { + return err + } + parts = append(parts, record) default: vprint.VV("UNKNOWN %T", sf.schema.Field(col).Type) } diff --git a/arrow.go b/arrow.go index 1d009b285..dac7dca36 100644 --- a/arrow.go +++ b/arrow.go @@ -194,6 +194,8 @@ func (st *basicTable) Get(column, row int) interface{} { case *arrow.Float64Type: v := chunk.(*array.Float64).Float64Values() return v[i] + case *arrow.StringType: + return chunk.(*array.String).Value(i) } return 0 } @@ -223,8 +225,10 @@ func builderFrom(mem memory.Allocator, dt arrow.DataType, size int64) array.Buil bldr = array.NewFloat32Builder(mem) case *arrow.Float64Type: bldr = array.NewFloat64Builder(mem) + case *arrow.StringType: + bldr = array.NewStringBuilder(mem) default: - panic(fmt.Errorf("npy2root: invalid Arrow type %v", dt)) + panic(fmt.Errorf("builderFrom: invalid Arrow type %v", dt)) } bldr.Reserve(int(size)) return bldr @@ -254,8 +258,10 @@ func appendData(bldr array.Builder, v interface{}) { bldr.Append(v.(float32)) case *array.Float64Builder: bldr.Append(v.(float64)) + case *array.StringBuilder: + bldr.Append(v.(string)) default: - panic(fmt.Errorf("npy2root: invalid Arrow builder type %T", bldr)) + panic(fmt.Errorf("appendData: invalid Arrow builder type %T", bldr)) } } diff --git a/ctl/dataframe-csv-loader.go b/ctl/dataframe-csv-loader.go index 10ccc9785..70cc45e3e 100644 --- a/ctl/dataframe-csv-loader.go +++ b/ctl/dataframe-csv-loader.go @@ -31,6 +31,7 @@ var ( func init() { gob.Register(arrow.PrimitiveTypes.Int64) gob.Register(arrow.PrimitiveTypes.Float64) + gob.Register(arrow.BinaryTypes.String) } // TODO(rdp): add refresh token to this as well @@ -154,7 +155,7 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) { } fields := make([]arrow.Field, 0) fields = append(fields, arrow.Field{Name: "_ID", Type: arrow.PrimitiveTypes.Int64}) - fileScanner := bufio.NewScanner(readFile) + fileScanner := bufio.NewScanner(readFile) // TODO(twg) 2023/01/11 need to convert to the go CSV reader for more robust string support fileScanner.Split(bufio.ScanLines) // need for really long csv lines var buf []byte @@ -176,6 +177,8 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) { fields = append(fields, arrow.Field{Name: name, Type: arrow.PrimitiveTypes.Int64}) } else if strings.HasSuffix(col, "__F") { fields = append(fields, arrow.Field{Name: name, Type: arrow.PrimitiveTypes.Float64}) + } else if strings.HasSuffix(col, "__S") { + fields = append(fields, arrow.Field{Name: name, Type: arrow.BinaryTypes.String}) } else { return errors.New("invalid format for type") } @@ -268,6 +271,8 @@ func (cmd *DataframeCsvLoaderCommand) Run(ctx context.Context) (err error) { continue } shardFile.SetFloatValue(i, shardRow, val) + case arrow.BinaryTypes.String: + shardFile.SetStringValue(i, shardRow, rec) default: return errors.New("unhandled arrow type type") } @@ -314,6 +319,11 @@ func (s *ShardDiff) SetFloatValue(col int, row int64, val float64) { s.columns[col] = append(slice, val) } +func (s *ShardDiff) SetStringValue(col int, row int64, val string) { + slice := s.columns[col].([]string) + s.columns[col] = append(slice, val) +} + func (s *ShardDiff) SetNulll(col int, row uint64) { s.null[pair{col: col, row: row}] = struct{}{} } @@ -329,6 +339,8 @@ func (s *ShardDiff) Setup(schema *arrow.Schema) { s.columns = append(s.columns, make([]int64, 0)) case arrow.PrimitiveTypes.Float64: s.columns = append(s.columns, make([]float64, 0)) + case arrow.BinaryTypes.String: + s.columns = append(s.columns, make([]string, 0)) } } } diff --git a/ctl/dataframe-csv-loader_test.go b/ctl/dataframe-csv-loader_test.go index 8280fc026..41df6ff7c 100644 --- a/ctl/dataframe-csv-loader_test.go +++ b/ctl/dataframe-csv-loader_test.go @@ -66,6 +66,33 @@ func TestDataframeCsvLoaderCommand(t *testing.T) { cm.Path = file.Name() cm.Index = index + err = cm.Run(ctx) + if err != nil { + t.Fatalf("DataframeCsvLoader Run doesn't work: %s", err) + } + }) + t.Run("strings", func(t *testing.T) { + cmLog := logger.NewStandardLogger(io.Discard) + cm := NewDataframeCsvLoaderCommand(cmLog) + file, err := testhook.TempFile(t, "import_string.csv") + if err != nil { + t.Fatalf("creating tempfile: %v", err) + } + _, err = file.Write([]byte("id,val__S\nA,ab\nB,cd\nC,ef")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } + ctx := context.Background() + if err != nil { + t.Fatal(err) + } + index := "strings" + cmd.API.CreateIndex(ctx, index, pilosa.IndexOptions{Keys: true}) + + cm.Host = cmd.API.Node().URI.HostPort() + cm.Path = file.Name() + cm.Index = index + err = cm.Run(ctx) if err != nil { t.Fatalf("DataframeCsvLoader Run doesn't work: %s", err) diff --git a/dataframe_test.go b/dataframe_test.go index a94bd2247..0cbc488b9 100644 --- a/dataframe_test.go +++ b/dataframe_test.go @@ -51,19 +51,20 @@ func TestExecutor_Apply(t *testing.T) { } t.Run("dataframe ingest", func(t *testing.T) { - // func (c *Client) ApplyDataframeChangeset(indexName string, cr *pilosa.ChangesetRequest, shard uint64) (map[string]interface{}, error) { cr := &pilosa.ChangesetRequest{} // for each row a list of columns cr.Columns = []interface{}{ []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, []int64{2, 4, 6, 8, 10, 12, 14, 16, 18, 20}, []float64{1, 1.414, 1.732, 2, 2.236, 2.449, 2.646, 2.828, 3, 3.162}, + []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J"}, } cr.ShardIds = []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} cr.SimpleSchema = []pilosa.NameType{ {Name: "_ID", DataType: arrow.PrimitiveTypes.Int64}, {Name: "ival", DataType: arrow.PrimitiveTypes.Int64}, {Name: "fval", DataType: arrow.PrimitiveTypes.Float64}, + {Name: "sval", DataType: arrow.BinaryTypes.String}, } shard := uint64(0) err := api.ApplyDataframeChangeset(ctx, indexName, cr, shard) @@ -72,7 +73,7 @@ func TestExecutor_Apply(t *testing.T) { } }) t.Run("dataframe schema", func(t *testing.T) { - expectedJSON := `[{"Name":"_ID","Type":"int64"},{"Name":"ival","Type":"int64"},{"Name":"fval","Type":"float64"}]` + expectedJSON := `[{"Name":"_ID","Type":"int64"},{"Name":"ival","Type":"int64"},{"Name":"fval","Type":"float64"},{"Name":"sval","Type":"utf8"}]` parts, err := api.GetDataframeSchema(ctx, indexName) if err != nil { t.Fatal(err) @@ -138,7 +139,7 @@ func TestExecutor_Apply(t *testing.T) { if res, err := api.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil { t.Fatal(err) } else { - expectedJSON := `{"Results":[{"_ID":[2,4,6],"fval":[1.414,2,2.449],"ival":[4,8,12]}],"Err":null,"Profile":null}` + expectedJSON := `{"Results":[{"_ID":[2,4,6],"fval":[1.414,2,2.449],"ival":[4,8,12],"sval":["B","D","F"]}],"Err":null,"Profile":null}` w := new(bytes.Buffer) if err := json.NewEncoder(w).Encode(res); err != nil { t.Fatal(err) @@ -165,6 +166,44 @@ func TestExecutor_Apply(t *testing.T) { } } }) + t.Run("dataframe ingest update", func(t *testing.T) { + cr := &pilosa.ChangesetRequest{} + // for each row a list of columns + cr.Columns = []interface{}{ + []int64{1}, + []int64{20}, + []float64{10}, + []string{"A2"}, + } + cr.ShardIds = []int64{1} + cr.SimpleSchema = []pilosa.NameType{ + {Name: "_ID", DataType: arrow.PrimitiveTypes.Int64}, + {Name: "ival", DataType: arrow.PrimitiveTypes.Int64}, + {Name: "fval", DataType: arrow.PrimitiveTypes.Float64}, + {Name: "sval", DataType: arrow.BinaryTypes.String}, + } + shard := uint64(0) + err := api.ApplyDataframeChangeset(ctx, indexName, cr, shard) + if err != nil { + t.Fatal(err) + } + }) + t.Run("dataframe arrow filter with header", func(t *testing.T) { + pql := `Arrow(ConstRow(columns=[1]),header=["ival","fval","sval"])` + if res, err := api.Query(ctx, &pilosa.QueryRequest{Index: indexName, Query: pql}); err != nil { + t.Fatal(err) + } else { + expectedJSON := `{"Results":[{"_ID":[1],"fval":[10],"ival":[20],"sval":["A2"]}],"Err":null,"Profile":null}` + w := new(bytes.Buffer) + if err := json.NewEncoder(w).Encode(res); err != nil { + t.Fatal(err) + } + got := strings.Trim(w.String(), "\t \n") + if strings.Compare(got, expectedJSON) != 0 { + t.Fatalf("expected: %v got: %v", expectedJSON, got) + } + } + }) t.Run("dataframe delete", func(t *testing.T) { err := api.DeleteDataframe(ctx, indexName) if err != nil { diff --git a/go.mod b/go.mod index 341712cb2..9710bf3c7 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ replace github.com/go-avro/avro => github.com/pilosa/avro v0.0.0-20200626214113- replace github.com/gomem/gomem => github.com/tgruben/gomem v0.0.0-20221021111114-79fdc77dcf61 -replace robpike.io/ivy => github.com/tgruben/ivy v0.0.0-20221107170120-634b546dcdac +replace robpike.io/ivy => github.com/tgruben/ivy v0.0.0-20230111144143-b80a659caeaf require ( github.com/CAFxX/gcnotifier v0.0.0-20220409005548-0153238b886a diff --git a/go.sum b/go.sum index 8c291fb52..7657c22ad 100644 --- a/go.sum +++ b/go.sum @@ -1058,8 +1058,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tgruben/gomem v0.0.0-20221021111114-79fdc77dcf61 h1:3RJ3IN/m4w4B7PeYF8PPOdGLHqu045LIxtymF4lQO7g= github.com/tgruben/gomem v0.0.0-20221021111114-79fdc77dcf61/go.mod h1:avNE+ynGJYvQNY+/5Gk6aHxtlOtt2aO6V9ShcONBvvI= -github.com/tgruben/ivy v0.0.0-20221107170120-634b546dcdac h1:xVEKycwTG+9q18T1zA+ESrkIMIMQMmlsGOFirh5VxT8= -github.com/tgruben/ivy v0.0.0-20221107170120-634b546dcdac/go.mod h1:/COPfnSdd23BhmpDXFKsYPk1kK2sJpUr21TsZ1nsgcg= +github.com/tgruben/ivy v0.0.0-20230111144143-b80a659caeaf h1:+bnHPov8gMIztSP1zxMbUXei8IiVNXbQre4+u9lpkQ0= +github.com/tgruben/ivy v0.0.0-20230111144143-b80a659caeaf/go.mod h1:/COPfnSdd23BhmpDXFKsYPk1kK2sJpUr21TsZ1nsgcg= github.com/tidwall/btree v0.3.0/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8= github.com/tidwall/btree v1.1.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= github.com/tidwall/buntdb v1.2.0/go.mod h1:XLza/dhlwzO6dc5o/KWor4kfZSt3BP8QV+77ZMKfI58= diff --git a/http_handler.go b/http_handler.go index 2bfe07cfd..d775554c3 100644 --- a/http_handler.go +++ b/http_handler.go @@ -4206,6 +4206,7 @@ func (h *Handler) handleGetHealth(w http.ResponseWriter, r *http.Request) { func init() { gob.Register(arrow.PrimitiveTypes.Int64) gob.Register(arrow.PrimitiveTypes.Float64) + gob.Register(arrow.BinaryTypes.String) } // EXPERIMENTAL API MAY CHANGE From fc1b8fdfb8a3070fb54703f741bbd0661f5796af Mon Sep 17 00:00:00 2001 From: rachithrr Date: Thu, 12 Jan 2023 18:01:06 -0500 Subject: [PATCH 03/15] FB-1827: Implement Len() (#2406) (cherry picked from commit f99be656dfdab0cfda3b34d74afa12e6b511d83d) --- sql3/planner/expression.go | 2 ++ sql3/planner/expressionanalyzercall.go | 2 ++ sql3/planner/inbuiltfunctionsstring.go | 32 ++++++++++++++++++++ sql3/test/defs/defs_string_functions.go | 39 +++++++++++++++++++++++++ 4 files changed, 75 insertions(+) diff --git a/sql3/planner/expression.go b/sql3/planner/expression.go index e0ad68c1f..2253719a0 100644 --- a/sql3/planner/expression.go +++ b/sql3/planner/expression.go @@ -1516,6 +1516,8 @@ func (n *callPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er return n.EvaluatePrefix(currentRow) case "SPACE": return n.EvaluateSpace(currentRow) + case "LEN": + return n.EvaluateLen(currentRow) default: return nil, sql3.NewErrInternalf("unhandled function name '%s'", n.name) } diff --git a/sql3/planner/expressionanalyzercall.go b/sql3/planner/expressionanalyzercall.go index a2b6d04b1..1b743b913 100644 --- a/sql3/planner/expressionanalyzercall.go +++ b/sql3/planner/expressionanalyzercall.go @@ -269,6 +269,8 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser return p.analyseFunctionPrefixSuffix(call, scope) case "SPACE": return p.analyseFunctionSpace(call, scope) + case "LEN": + return p.analyseFunctionLen(call, scope) default: return nil, sql3.NewErrCallUnknownFunction(call.Name.NamePos.Line, call.Name.NamePos.Column, call.Name.Name) } diff --git a/sql3/planner/inbuiltfunctionsstring.go b/sql3/planner/inbuiltfunctionsstring.go index fda45b18d..52bb71e28 100644 --- a/sql3/planner/inbuiltfunctionsstring.go +++ b/sql3/planner/inbuiltfunctionsstring.go @@ -195,6 +195,18 @@ func (p *ExecutionPlanner) analyseFunctionSpace(call *parser.Call, scope parser. return call, nil } +func (p *ExecutionPlanner) analyseFunctionLen(call *parser.Call, scope parser.Statement) (parser.Expr, error) { + if len(call.Args) != 1 { + return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 1, len(call.Args)) + } + + if !typeIsString(call.Args[0].DataType()) && !typeIsVoid(call.Args[0].DataType()) { + return nil, sql3.NewErrStringExpressionExpected(call.Args[0].Pos().Line, call.Args[0].Pos().Column) + } + call.ResultDataType = parser.NewDataTypeInt() + return call, nil +} + // reverses the string func (n *callPlanExpression) EvaluateReverse(currentRow []interface{}) (interface{}, error) { argEval, err := n.args[0].Evaluate(currentRow) @@ -264,6 +276,10 @@ func (n *callPlanExpression) EvaluateChar(currentRow []interface{}) (interface{} if !ok { return 0, sql3.NewErrInternalf("unexpected type converion %T", argEval) } + // ascii range is [0-255] + if intArg < 0 || intArg > 255 { + return nil, sql3.NewErrValueOutOfRange(0, 0, intArg) + } // Return the character that corresponds to the integer value return string(rune(intArg)), nil @@ -580,3 +596,19 @@ func (n *callPlanExpression) EvaluateSpace(currentRow []interface{}) (interface{ } return spaces, nil } + +func (n *callPlanExpression) EvaluateLen(currentRow []interface{}) (interface{}, error) { + argEval, err := n.args[0].Evaluate(currentRow) + if err != nil { + return nil, err + } + if argEval == nil { + return nil, nil + } + stringArg, ok := argEval.(string) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type converion %T", argEval) + } + + return int64(len([]rune(stringArg))), nil +} diff --git a/sql3/test/defs/defs_string_functions.go b/sql3/test/defs/defs_string_functions.go index 1790fd8e4..54529285f 100644 --- a/sql3/test/defs/defs_string_functions.go +++ b/sql3/test/defs/defs_string_functions.go @@ -727,5 +727,44 @@ var stringScalarFunctionsTests = TableTest{ ), Compare: CompareExactOrdered, }, + { + name: "LenNull", + SQLs: sqls( + "select len(null)", + ), + ExpHdrs: hdrs( + hdr("", fldTypeInt), + ), + ExpRows: rows( + row(nil), + ), + Compare: CompareExactOrdered, + }, + { + name: "LenString", + SQLs: sqls( + "select len(' length ')", + ), + ExpHdrs: hdrs( + hdr("", fldTypeInt), + ), + ExpRows: rows( + row(int64(9)), + ), + Compare: CompareExactOrdered, + }, + { + name: "LenChar", + SQLs: sqls( + "select len(char(114))", + ), + ExpHdrs: hdrs( + hdr("", fldTypeInt), + ), + ExpRows: rows( + row(int64(1)), + ), + Compare: CompareExactOrdered, + }, }, } From 5d025e399bf43bcbc02eb93ae1aa3d42ffe754fc Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula <85502298+pokeeffe-molecula@users.noreply.github.com> Date: Fri, 13 Jan 2023 18:57:31 -0600 Subject: [PATCH 04/15] implement CREATE/ALTER/DROP VIEW (fb-1592) (#2408) * implement CREATE/ALTER/DROP VIEW * fixed failing test * another failing test * fixed some broken serverless tests (cherry picked from commit c620aae350403f21cd4d8647f677b1eac3695b3a) --- dax/test/dax/dax_test.go | 3 + schema.go | 3 + sql3/errors.go | 26 ++ sql3/parser/ast.go | 92 +++++-- sql3/parser/ast_test.go | 30 ++- sql3/parser/parser.go | 112 ++++++--- sql3/parser/parser_test.go | 29 +-- sql3/parser/walk.go | 6 +- sql3/planner/compilealtertable.go | 2 +- sql3/planner/compilebulkinsert.go | 6 +- sql3/planner/compilecreateview.go | 64 +++++ sql3/planner/compiledelete.go | 2 +- sql3/planner/compiledroptable.go | 2 +- sql3/planner/compiledropview.go | 23 ++ sql3/planner/compileinsert.go | 4 +- sql3/planner/compileselect.go | 113 ++++++--- sql3/planner/compileshow.go | 4 +- sql3/planner/executionplanner.go | 18 ++ sql3/planner/executionplannersystemtables.go | 4 +- sql3/planner/opalterview.go | 97 ++++++++ sql3/planner/opcreateview.go | 120 +++++++++ sql3/planner/opdropview.go | 102 ++++++++ sql3/planner/oppqldistinctscan.go | 2 +- sql3/planner/oppqltablescan.go | 2 +- sql3/planner/planoptimizer.go | 2 + sql3/planner/systemobjects.go | 247 +++++++++++++++++++ sql3/sql_complex_test.go | 5 +- sql3/test/defs/defs.go | 1 + sql3/test/defs/defs_views.go | 126 ++++++++++ 29 files changed, 1125 insertions(+), 122 deletions(-) create mode 100644 sql3/planner/compilecreateview.go create mode 100644 sql3/planner/compiledropview.go create mode 100644 sql3/planner/opalterview.go create mode 100644 sql3/planner/opcreateview.go create mode 100644 sql3/planner/opdropview.go create mode 100644 sql3/planner/systemobjects.go create mode 100644 sql3/test/defs/defs_views.go diff --git a/dax/test/dax/dax_test.go b/dax/test/dax/dax_test.go index a5f026412..87b510a3b 100644 --- a/dax/test/dax/dax_test.go +++ b/dax/test/dax/dax_test.go @@ -112,6 +112,9 @@ func TestDAXIntegration(t *testing.T) { "alterTable/alterTableBadTable", // looks like table does not exist is a different error in DAX "top-tests/test-1", // don't know why this is failing at all "delete_tests", + "viewtests/drop-view", // drop view does a delete + "viewtests/drop-view-if-exists-after-drop", + "viewtests/select-view-after-drop", } doSkip := func(name string) bool { diff --git a/schema.go b/schema.go index 9c9a4a426..36fa67613 100644 --- a/schema.go +++ b/schema.go @@ -28,6 +28,9 @@ func NewOnPremSchema(api *API) *onPremSchema { func (s *onPremSchema) TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error) { idx, err := s.api.IndexInfo(context.Background(), string(tname)) if err != nil { + if err == ErrIndexNotFound { + return nil, dax.NewErrTableNameDoesNotExist(tname) + } return nil, errors.Wrapf(err, "getting index info for table name: %s", tname) } diff --git a/sql3/errors.go b/sql3/errors.go index 10c5ceff3..aab3a4218 100644 --- a/sql3/errors.go +++ b/sql3/errors.go @@ -79,6 +79,11 @@ const ( ErrTableColumnNotFound errors.Code = "ErrTableColumnNotFound" ErrInvalidKeyPartitionsValue errors.Code = "ErrInvalidKeyPartitionsValue" + ErrTableOrViewNotFound errors.Code = "ErrTableOrViewNotFound" + + ErrViewExists errors.Code = "ErrViewExists" + ErrViewNotFound errors.Code = "ErrViewNotFound" + ErrBadColumnConstraint errors.Code = "ErrBadColumnConstraint" ErrConflictingColumnConstraint errors.Code = "ErrConflictingColumnConstraint" @@ -507,6 +512,13 @@ func NewErrTableNotFound(line, col int, tableName string) error { ) } +func NewErrTableOrViewNotFound(line, col int, tableName string) error { + return errors.New( + ErrTableOrViewNotFound, + fmt.Sprintf("[%d:%d] table or view '%s' not found", line, col, tableName), + ) +} + func NewErrTableExists(line, col int, tableName string) error { return errors.New( ErrTableExists, @@ -535,6 +547,20 @@ func NewErrInvalidKeyPartitionsValue(line, col int, keypartitions int64) error { ) } +func NewErrViewNotFound(line, col int, viewName string) error { + return errors.New( + ErrViewNotFound, + fmt.Sprintf("[%d:%d] view '%s' not found", line, col, viewName), + ) +} + +func NewErrViewExists(line, col int, viewName string) error { + return errors.New( + ErrViewExists, + fmt.Sprintf("[%d:%d] view '%s' already exists", line, col, viewName), + ) +} + func NewErrBadColumnConstraint(line, col int, constraint, columnType string) error { return errors.New( ErrBadColumnConstraint, diff --git a/sql3/parser/ast.go b/sql3/parser/ast.go index 0c90d7c78..c071164c1 100644 --- a/sql3/parser/ast.go +++ b/sql3/parser/ast.go @@ -36,6 +36,7 @@ func (*CreateIndexStatement) node() {} func (*CreateTableStatement) node() {} func (*CreateFunctionStatement) node() {} func (*CreateViewStatement) node() {} +func (*AlterViewStatement) node() {} func (*DateLit) node() {} func (*DefaultConstraint) node() {} func (*DeleteStatement) node() {} @@ -111,6 +112,7 @@ func (*CreateIndexStatement) stmt() {} func (*CreateTableStatement) stmt() {} func (*CreateFunctionStatement) stmt() {} func (*CreateViewStatement) stmt() {} +func (*AlterViewStatement) stmt() {} func (*DeleteStatement) stmt() {} func (*DropIndexStatement) stmt() {} func (*DropTableStatement) stmt() {} @@ -2487,17 +2489,18 @@ func (s *DropTableStatement) String() string { } type CreateViewStatement struct { - Create Pos // position of CREATE keyword - View Pos // position of VIEW keyword - If Pos // position of IF keyword - IfNot Pos // position of NOT keyword after IF - IfNotExists Pos // position of EXISTS keyword after IF NOT - Name *Ident // view name - Lparen Pos // position of column list left paren - Columns []*Ident // column list - Rparen Pos // position of column list right paren - As Pos // position of AS keyword - Select *SelectStatement // source statement + Create Pos // position of CREATE keyword + View Pos // position of VIEW keyword + If Pos // position of IF keyword + IfNot Pos // position of NOT keyword after IF + IfNotExists Pos // position of EXISTS keyword after IF NOT + Name *Ident // view name + // TODO(pok) - we'll do this later - see note in parseCompileView() + // Lparen Pos // position of column list left paren + // Columns []*Ident // column list + // Rparen Pos // position of column list right paren + As Pos // position of AS keyword + Select *SelectStatement // source statement } // Clone returns a deep copy of s. @@ -2507,7 +2510,7 @@ func (s *CreateViewStatement) Clone() *CreateViewStatement { } other := *s other.Name = s.Name.Clone() - other.Columns = cloneIdents(s.Columns) + // other.Columns = cloneIdents(s.Columns) other.Select = s.Select.Clone() return &other } @@ -2521,22 +2524,67 @@ func (s *CreateViewStatement) String() string { } fmt.Fprintf(&buf, " %s", s.Name.String()) - if len(s.Columns) > 0 { - buf.WriteString(" (") - for i, col := range s.Columns { - if i != 0 { - buf.WriteString(", ") - } - buf.WriteString(col.String()) - } - buf.WriteString(")") - } + // if len(s.Columns) > 0 { + // buf.WriteString(" (") + // for i, col := range s.Columns { + // if i != 0 { + // buf.WriteString(", ") + // } + // buf.WriteString(col.String()) + // } + // buf.WriteString(")") + // } fmt.Fprintf(&buf, " AS %s", s.Select.String()) return buf.String() } +type AlterViewStatement struct { + Alter Pos // position of CREATE keyword + View Pos // position of VIEW keyword + Name *Ident // view name + + // TODO(pok) - we'll do this later - see note in parseCompileView() + // Lparen Pos // position of column list left paren + // Columns []*Ident // column list + // Rparen Pos // position of column list right paren + As Pos // position of AS keyword + Select *SelectStatement // source statement +} + +// Clone returns a deep copy of s. +func (s *AlterViewStatement) Clone() *AlterViewStatement { + if s == nil { + return nil + } + other := *s + other.Name = s.Name.Clone() + // other.Columns = cloneIdents(s.Columns) + other.Select = s.Select.Clone() + return &other +} + +// String returns the string representation of the statement. +func (s *AlterViewStatement) String() string { + var buf bytes.Buffer + buf.WriteString("ALTER VIEW") + fmt.Fprintf(&buf, " %s", s.Name.String()) + + // if len(s.Columns) > 0 { + // buf.WriteString(" (") + // for i, col := range s.Columns { + // if i != 0 { + // buf.WriteString(", ") + // } + // buf.WriteString(col.String()) + // } + // buf.WriteString(")") + // } + fmt.Fprintf(&buf, " AS %s", s.Select.String()) + return buf.String() +} + type DropViewStatement struct { Drop Pos // position of DROP keyword View Pos // position of VIEW keyword diff --git a/sql3/parser/ast_test.go b/sql3/parser/ast_test.go index 092b21e6c..c326e055b 100644 --- a/sql3/parser/ast_test.go +++ b/sql3/parser/ast_test.go @@ -395,14 +395,15 @@ func TestCreateFunctionStatement_String(t *testing.T) { func TestCreateViewStatement_String(t *testing.T) { AssertStatementStringer(t, &parser.CreateViewStatement{ Name: &parser.Ident{Name: "vw"}, - Columns: []*parser.Ident{ - {Name: "x"}, - {Name: "y"}, - }, + // Columns: []*parser.Ident{ + // {Name: "x"}, + // {Name: "y"}, + // }, Select: &parser.SelectStatement{ Columns: []*parser.ResultColumn{{Star: pos(0)}}, }, - }, `CREATE VIEW vw (x, y) AS SELECT *`) + //}, `CREATE VIEW vw (x, y) AS SELECT *`) + }, `CREATE VIEW vw AS SELECT *`) AssertStatementStringer(t, &parser.CreateViewStatement{ IfNotExists: pos(0), @@ -413,6 +414,20 @@ func TestCreateViewStatement_String(t *testing.T) { }, `CREATE VIEW IF NOT EXISTS vw AS SELECT *`) } +func TestAlterViewStatement_String(t *testing.T) { + AssertStatementStringer(t, &parser.AlterViewStatement{ + Name: &parser.Ident{Name: "vw"}, + // Columns: []*parser.Ident{ + // {Name: "x"}, + // {Name: "y"}, + // }, + Select: &parser.SelectStatement{ + Columns: []*parser.ResultColumn{{Star: pos(0)}}, + }, + //}, `CREATE VIEW vw (x, y) AS SELECT *`) + }, `ALTER VIEW vw AS SELECT *`) +} + func TestDeleteStatement_String(t *testing.T) { AssertStatementStringer(t, &parser.DeleteStatement{ TableName: &parser.QualifiedTableName{Name: &parser.Ident{Name: "tbl"}, Alias: &parser.Ident{Name: "tbl2"}}, @@ -487,15 +502,14 @@ func TestDropTriggerStatement_String(t *testing.T) { } func TestDropViewStatement_String(t *testing.T) { - t.Skip("DROP VIEW is currently disabled in the parser") AssertStatementStringer(t, &parser.DropViewStatement{ Name: &parser.Ident{Name: "vw"}, - }, `DROP VIEW "vw"`) + }, `DROP VIEW vw`) AssertStatementStringer(t, &parser.DropViewStatement{ IfExists: pos(0), Name: &parser.Ident{Name: "vw"}, - }, `DROP VIEW IF EXISTS "vw"`) + }, `DROP VIEW IF EXISTS vw`) } func TestExplainStatement_String(t *testing.T) { diff --git a/sql3/parser/parser.go b/sql3/parser/parser.go index 3f19647c8..af27ab6e0 100644 --- a/sql3/parser/parser.go +++ b/sql3/parser/parser.go @@ -98,17 +98,7 @@ func (p *Parser) parseNonExplainStatement() (Statement, error) { //case ANALYZE: // return p.parseAnalyzeStatement() case ALTER: - return p.parseAlterTableStatement() - // case BEGIN: - // return p.parseBeginStatement() - // case COMMIT, END: - // return p.parseCommitStatement() - // case ROLLBACK: - // return p.parseRollbackStatement() - // case SAVEPOINT: - // return p.parseSavepointStatement() - // case RELEASE: - // return p.parseReleaseStatement() + return p.parseAlterStatement() case BULK: return p.parseBulkInsertStatement() case CREATE: @@ -337,6 +327,20 @@ func (p *Parser) parseCreateStatement() (Statement, error) { } } +func (p *Parser) parseAlterStatement() (Statement, error) { + assert(p.peek() == ALTER) + pos, tok, _ := p.scan() + + switch p.peek() { + case TABLE: + return p.parseAlterTableStatement(pos) + case VIEW: + return p.parseAlterViewStatement(pos) + default: + return nil, p.errorExpected(pos, tok, "TABLE or VIEW") + } +} + func (p *Parser) parseDropStatement() (Statement, error) { assert(p.peek() == DROP) pos, tok, _ := p.scan() @@ -1054,25 +1058,73 @@ func (p *Parser) parseCreateViewStatement(createPos Pos) (_ *CreateViewStatement return &stmt, err } + // TODO(pok) - we'll do this later - right now views are implemented as + // jit compiled from text, which makes implementing these columns a pain + // when we can pre-compile a plan op subgraph and stored it, we can put this + // back in // Parse optional column list. - if p.peek() == LP { - stmt.Lparen, _, _ = p.scan() - for { - col, err := p.parseIdent("column name") - if err != nil { - return &stmt, err - } - stmt.Columns = append(stmt.Columns, col) + // if p.peek() == LP { + // stmt.Lparen, _, _ = p.scan() + // for { + // col, err := p.parseIdent("column name") + // if err != nil { + // return &stmt, err + // } + // stmt.Columns = append(stmt.Columns, col) - if p.peek() == RP { - break - } else if p.peek() != COMMA { - return &stmt, p.errorExpected(p.pos, p.tok, "comma or right paren") - } - p.scan() - } - stmt.Rparen, _, _ = p.scan() + // if p.peek() == RP { + // break + // } else if p.peek() != COMMA { + // return &stmt, p.errorExpected(p.pos, p.tok, "comma or right paren") + // } + // p.scan() + // } + // stmt.Rparen, _, _ = p.scan() + // } + + // Parse "AS select-stmt" + if p.peek() != AS { + return &stmt, p.errorExpected(p.pos, p.tok, "AS") } + stmt.As, _, _ = p.scan() + if stmt.Select, err = p.parseSelectStatement(false, nil); err != nil { + return &stmt, err + } + return &stmt, nil +} + +func (p *Parser) parseAlterViewStatement(alterPos Pos) (_ *AlterViewStatement, err error) { + var stmt AlterViewStatement + stmt.Alter = alterPos + if p.peek() != VIEW { + return &stmt, p.errorExpected(p.pos, p.tok, "VIEW") + } + stmt.View, _, _ = p.scan() + + if stmt.Name, err = p.parseIdent("view name"); err != nil { + return &stmt, err + } + + // TODO(pok) - we'll do this later - see note in parseCompileView() + // Parse optional column list. + // if p.peek() == LP { + // stmt.Lparen, _, _ = p.scan() + // for { + // col, err := p.parseIdent("column name") + // if err != nil { + // return &stmt, err + // } + // stmt.Columns = append(stmt.Columns, col) + + // if p.peek() == RP { + // break + // } else if p.peek() != COMMA { + // return &stmt, p.errorExpected(p.pos, p.tok, "comma or right paren") + // } + // p.scan() + // } + // stmt.Rparen, _, _ = p.scan() + // } // Parse "AS select-stmt" if p.peek() != AS { @@ -3225,11 +3277,9 @@ func (p *Parser) parseIntegerLiteral(desc string) (*IntegerLit, error) { } } -func (p *Parser) parseAlterTableStatement() (_ *AlterTableStatement, err error) { - assert(p.peek() == ALTER) - +func (p *Parser) parseAlterTableStatement(alterPos Pos) (_ *AlterTableStatement, err error) { var stmt AlterTableStatement - stmt.Alter, _, _ = p.scan() + stmt.Alter = alterPos if p.peek() != TABLE { return &stmt, p.errorExpected(p.pos, p.tok, "TABLE") } diff --git a/sql3/parser/parser_test.go b/sql3/parser/parser_test.go index 74ecaf725..748ddf0f4 100644 --- a/sql3/parser/parser_test.go +++ b/sql3/parser/parser_test.go @@ -427,7 +427,7 @@ func TestParser_ParseAlterStatement(t *testing.T) { DropColumnName: &parser.Ident{NamePos: pos(28), Name: "col"}, }) - AssertParseStatementError(t, `ALTER`, `1:5: expected TABLE, found 'EOF'`) + AssertParseStatementError(t, `ALTER`, `1:1: expected TABLE or VIEW`) AssertParseStatementError(t, `ALTER TABLE`, `1:11: expected table name, found 'EOF'`) AssertParseStatementError(t, `ALTER TABLE tbl`, `1:15: expected ADD, DROP or RENAME, found 'EOF'`) AssertParseStatementError(t, `ALTER TABLE tbl RENAME`, `1:22: expected COLUMN keyword or column name, found 'EOF'`) @@ -1566,22 +1566,23 @@ func TestParser_ParseStatement(t *testing.T) { }) t.Run("CreateView", func(t *testing.T) { - AssertParseStatement(t, `CREATE VIEW vw (col1, col2) AS SELECT x, y`, &parser.CreateViewStatement{ + //AssertParseStatement(t, `CREATE VIEW vw (col1, col2) AS SELECT x, y`, &parser.CreateViewStatement{ + AssertParseStatement(t, `CREATE VIEW vw AS SELECT x, y`, &parser.CreateViewStatement{ Create: pos(0), View: pos(7), Name: &parser.Ident{NamePos: pos(12), Name: "vw"}, - Lparen: pos(15), - Columns: []*parser.Ident{ - {NamePos: pos(16), Name: "col1"}, - {NamePos: pos(22), Name: "col2"}, - }, - Rparen: pos(26), - As: pos(28), + // Lparen: pos(15), + // Columns: []*parser.Ident{ + // {NamePos: pos(16), Name: "col1"}, + // {NamePos: pos(22), Name: "col2"}, + // }, + // Rparen: pos(26), + As: pos(15), Select: &parser.SelectStatement{ - Select: pos(31), + Select: pos(18), Columns: []*parser.ResultColumn{ - {Expr: &parser.Ident{NamePos: pos(38), Name: "x"}}, - {Expr: &parser.Ident{NamePos: pos(41), Name: "y"}}, + {Expr: &parser.Ident{NamePos: pos(25), Name: "x"}}, + {Expr: &parser.Ident{NamePos: pos(28), Name: "y"}}, }, }, }) @@ -1616,8 +1617,8 @@ func TestParser_ParseStatement(t *testing.T) { AssertParseStatementError(t, `CREATE VIEW IF`, `1:14: expected NOT, found 'EOF'`) AssertParseStatementError(t, `CREATE VIEW IF NOT`, `1:18: expected EXISTS, found 'EOF'`) AssertParseStatementError(t, `CREATE VIEW vw`, `1:14: expected AS, found 'EOF'`) - AssertParseStatementError(t, `CREATE VIEW vw (`, `1:16: expected column name, found 'EOF'`) - AssertParseStatementError(t, `CREATE VIEW vw (x`, `1:17: expected comma or right paren, found 'EOF'`) + //AssertParseStatementError(t, `CREATE VIEW vw (`, `1:16: expected column name, found 'EOF'`) + //AssertParseStatementError(t, `CREATE VIEW vw (x`, `1:17: expected comma or right paren, found 'EOF'`) AssertParseStatementError(t, `CREATE VIEW vw AS`, `1:17: expected SELECT, found 'EOF'`) AssertParseStatementError(t, `CREATE VIEW vw AS SELECT`, `1:24: expected expression, found 'EOF'`) }) diff --git a/sql3/parser/walk.go b/sql3/parser/walk.go index 7db4c35a4..def04823d 100644 --- a/sql3/parser/walk.go +++ b/sql3/parser/walk.go @@ -114,9 +114,9 @@ func walk(v Visitor, node Node) (_ Node, err error) { if err := walkIdent(v, &n.Name); err != nil { return node, err } - if err := walkIdentList(v, n.Columns); err != nil { - return node, err - } + // if err := walkIdentList(v, n.Columns); err != nil { + // return node, err + // } if n.Select != nil { if sel, err := walk(v, n.Select); err != nil { return node, err diff --git a/sql3/planner/compilealtertable.go b/sql3/planner/compilealtertable.go index 3031242b7..30767a1f6 100644 --- a/sql3/planner/compilealtertable.go +++ b/sql3/planner/compilealtertable.go @@ -31,7 +31,7 @@ func (p *ExecutionPlanner) compileAlterTableStatement(stmt *parser.AlterTableSta tname := dax.TableName(tableName) tbl, err := p.schemaAPI.TableByName(context.Background(), tname) if err != nil { - if errors.Is(err, pilosa.ErrIndexNotFound) { + if isTableNotFoundError(err) { return nil, sql3.NewErrTableNotFound(stmt.Name.NamePos.Line, stmt.Name.NamePos.Column, tableName) } return nil, err diff --git a/sql3/planner/compilebulkinsert.go b/sql3/planner/compilebulkinsert.go index d934dc61b..466df5040 100644 --- a/sql3/planner/compilebulkinsert.go +++ b/sql3/planner/compilebulkinsert.go @@ -25,7 +25,7 @@ func (p *ExecutionPlanner) compileBulkInsertStatement(stmt *parser.BulkInsertSta tname := dax.TableName(tableName) tbl, err := p.schemaAPI.TableByName(context.Background(), tname) if err != nil { - if errors.Is(err, pilosa.ErrIndexNotFound) { + if isTableNotFoundError(err) { return nil, sql3.NewErrTableNotFound(stmt.Table.NamePos.Line, stmt.Table.NamePos.Column, tableName) } return nil, err @@ -63,7 +63,7 @@ func (p *ExecutionPlanner) compileBulkInsertStatement(stmt *parser.BulkInsertSta switch strings.ToUpper(options.input) { case "FILE": // file should exist - if _, err := os.Stat(options.sourceData); errors.Is(err, os.ErrNotExist) { + if _, err := os.Stat(options.sourceData); goerrors.Is(err, os.ErrNotExist) { return nil, sql3.NewErrReadingDatasource(stmt.DataSource.Pos().Line, stmt.DataSource.Pos().Column, options.sourceData, fmt.Sprintf("file '%s' does not exist", options.sourceData)) } case "URL", "STREAM": @@ -163,7 +163,7 @@ func (p *ExecutionPlanner) analyzeBulkInsertStatement(stmt *parser.BulkInsertSta tname := dax.TableName(tableName) tbl, err := p.schemaAPI.TableByName(context.Background(), tname) if err != nil { - if errors.Is(err, pilosa.ErrIndexNotFound) { + if isTableNotFoundError(err) { return sql3.NewErrTableNotFound(stmt.Table.NamePos.Line, stmt.Table.NamePos.Column, tableName) } return err diff --git a/sql3/planner/compilecreateview.go b/sql3/planner/compilecreateview.go new file mode 100644 index 000000000..d3a554ee7 --- /dev/null +++ b/sql3/planner/compilecreateview.go @@ -0,0 +1,64 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "github.com/molecula/featurebase/v3/sql3/parser" + "github.com/molecula/featurebase/v3/sql3/planner/types" +) + +// compileCreateViewStatement compiles a parser.CreateViewStatement AST into a PlanOperator +func (p *ExecutionPlanner) compileCreateViewStatement(stmt *parser.CreateViewStatement) (types.PlanOperator, error) { + viewName := parser.IdentName(stmt.Name) + view := &viewSystemObject{ + name: viewName, + } + + // compile select + _, err := p.compileSelectStatement(stmt.Select, true) + if err != nil { + return nil, err + } + view.statement = stmt.Select.String() + + query := NewPlanOpQuery(p, NewPlanOpCreateView(p, stmt.IfNotExists.IsValid(), view), p.sql) + return query, nil +} + +// compileAlterViewStatement compiles a parser.AlterViewStatement AST into a PlanOperator +func (p *ExecutionPlanner) compileAlterViewStatement(stmt *parser.AlterViewStatement) (types.PlanOperator, error) { + viewName := parser.IdentName(stmt.Name) + view := &viewSystemObject{ + name: viewName, + } + + // compile select + _, err := p.compileSelectStatement(stmt.Select, true) + if err != nil { + return nil, err + } + view.statement = stmt.Select.String() + + query := NewPlanOpQuery(p, NewPlanOpAlterView(p, view), p.sql) + return query, nil +} + +func (p *ExecutionPlanner) analyzeCreateViewStatement(stmt *parser.CreateViewStatement) error { + //analyze the select + _, err := p.analyzeSelectStatement(stmt.Select) + if err != nil { + return err + } + + return nil +} + +func (p *ExecutionPlanner) analyzeAlterViewStatement(stmt *parser.AlterViewStatement) error { + //analyze the select + _, err := p.analyzeSelectStatement(stmt.Select) + if err != nil { + return err + } + + return nil +} diff --git a/sql3/planner/compiledelete.go b/sql3/planner/compiledelete.go index 2e253d4aa..97624b943 100644 --- a/sql3/planner/compiledelete.go +++ b/sql3/planner/compiledelete.go @@ -54,7 +54,7 @@ func (p *ExecutionPlanner) compileDeleteStatement(stmt *parser.DeleteStatement) func (p *ExecutionPlanner) analyzeDeleteStatement(stmt *parser.DeleteStatement) error { - err := p.analyzeSource(stmt.Source, stmt) + _, err := p.analyzeSource(stmt.Source, stmt) if err != nil { return err } diff --git a/sql3/planner/compiledroptable.go b/sql3/planner/compiledroptable.go index 48a147c33..2d70b8cfd 100644 --- a/sql3/planner/compiledroptable.go +++ b/sql3/planner/compiledroptable.go @@ -20,7 +20,7 @@ func (p *ExecutionPlanner) compileDropTableStatement(stmt *parser.DropTableState tname := dax.TableName(tableName) tbl, err := p.schemaAPI.TableByName(context.Background(), tname) if err != nil { - if errors.Is(err, pilosa.ErrIndexNotFound) { + if isTableNotFoundError(err) { return nil, sql3.NewErrTableNotFound(stmt.Name.NamePos.Line, stmt.Name.NamePos.Column, tableName) } return nil, err diff --git a/sql3/planner/compiledropview.go b/sql3/planner/compiledropview.go new file mode 100644 index 000000000..819c84fac --- /dev/null +++ b/sql3/planner/compiledropview.go @@ -0,0 +1,23 @@ +// Copyright 2023 Molecula Corp. All rights reserved. + +package planner + +import ( + "github.com/molecula/featurebase/v3/sql3" + "github.com/molecula/featurebase/v3/sql3/parser" + "github.com/molecula/featurebase/v3/sql3/planner/types" +) + +// compileDropViewStatement compiles a DROP VIEW statement into a PlanOperator. +func (p *ExecutionPlanner) compileDropViewStatement(stmt *parser.DropViewStatement) (_ types.PlanOperator, err error) { + viewName := parser.IdentName(stmt.Name) + v, err := p.getViewByName(viewName) + if err != nil { + return nil, err + } + if v == nil && !stmt.IfExists.IsValid() { + return nil, sql3.NewErrViewNotFound(0, 0, viewName) + } + + return NewPlanOpQuery(p, NewPlanOpDropView(p, stmt.IfExists.IsValid(), viewName), p.sql), nil +} diff --git a/sql3/planner/compileinsert.go b/sql3/planner/compileinsert.go index 4815e746c..06eaf11af 100644 --- a/sql3/planner/compileinsert.go +++ b/sql3/planner/compileinsert.go @@ -24,7 +24,7 @@ func (p *ExecutionPlanner) compileInsertStatement(stmt *parser.InsertStatement) tname := dax.TableName(tableName) tbl, err := p.schemaAPI.TableByName(context.Background(), tname) if err != nil { - if errors.Is(err, pilosa.ErrIndexNotFound) { + if isTableNotFoundError(err) { return nil, sql3.NewErrTableNotFound(stmt.Table.NamePos.Line, stmt.Table.NamePos.Column, tableName) } return nil, err @@ -79,7 +79,7 @@ func (p *ExecutionPlanner) analyzeInsertStatement(stmt *parser.InsertStatement) tname := dax.TableName(tableName) tbl, err := p.schemaAPI.TableByName(context.Background(), tname) if err != nil { - if errors.Is(err, pilosa.ErrIndexNotFound) { + if isTableNotFoundError(err) { return sql3.NewErrTableNotFound(stmt.Table.NamePos.Line, stmt.Table.NamePos.Column, tableName) } return err diff --git a/sql3/planner/compileselect.go b/sql3/planner/compileselect.go index 02d5efdec..0a112bed1 100644 --- a/sql3/planner/compileselect.go +++ b/sql3/planner/compileselect.go @@ -366,57 +366,97 @@ func (p *ExecutionPlanner) compileSource(scope *PlanOpQuery, source parser.Sourc } } -func (p *ExecutionPlanner) analyzeSource(source parser.Source, scope parser.Statement) error { +func (p *ExecutionPlanner) analyzeSource(source parser.Source, scope parser.Statement) (parser.Source, error) { if source == nil { - return nil + return nil, nil } switch source := source.(type) { case *parser.JoinClause: - err := p.analyzeSource(source.X, scope) + x, err := p.analyzeSource(source.X, scope) if err != nil { - return err + return nil, err } - err = p.analyzeSource(source.Y, scope) + y, err := p.analyzeSource(source.Y, scope) if err != nil { - return err + return nil, err } if source.Constraint != nil { switch join := source.Constraint.(type) { case *parser.OnConstraint: ex, err := p.analyzeExpression(join.X, scope) if err != nil { - return err + return nil, err } join.X = ex default: - return sql3.NewErrInternalf("unexpected constraint type '%T'", join) + return nil, sql3.NewErrInternalf("unexpected constraint type '%T'", join) } } - return nil + source.X = x + source.Y = y + return source, nil case *parser.ParenSource: - err := p.analyzeSource(source.X, scope) + x, err := p.analyzeSource(source.X, scope) if err != nil { - return err + return nil, err } - return nil + source.X = x + return source, nil case *parser.QualifiedTableName: + + objectName := parser.IdentName(source.Name) + + // check views first + view, err := p.getViewByName(objectName) + if err != nil { + return nil, err + } + + // if view is not null, it exists + if view != nil { + // parse the select statement + ast, err := parser.NewParser(strings.NewReader(view.statement)).ParseStatement() + if err != nil { + return nil, err + } + sel, ok := ast.(*parser.SelectStatement) + if !ok { + return nil, sql3.NewErrInternalf("unexpected ast type") + } + // analyze the select statement + expr, err := p.analyzeSelectStatement(sel) + if err != nil { + return nil, err + } + selExpr, ok := expr.(*parser.SelectStatement) + if !ok { + return nil, sql3.NewErrInternalf("unexpected analyzed type") + } + + // rewrite as a paren source with the select + paren := &parser.ParenSource{ + X: selExpr, + Alias: source.Alias, + } + return paren, nil + } + // check table exists - tableName := parser.IdentName(source.Name) - tname := dax.TableName(tableName) + tname := dax.TableName(objectName) tbl, err := p.schemaAPI.TableByName(context.Background(), tname) if err != nil { - if errors.Is(err, pilosa.ErrIndexNotFound) { - return sql3.NewErrTableNotFound(source.Name.NamePos.Line, source.Name.NamePos.Column, tableName) + if isTableNotFoundError(err) { + return nil, sql3.NewErrTableOrViewNotFound(source.Name.NamePos.Line, source.Name.NamePos.Column, objectName) } - return err + return nil, err } // populate the output columns from the source for i, fld := range tbl.Fields { soc := &parser.SourceOutputColumn{ - TableName: tableName, + TableName: objectName, ColumnName: string(fld.Name), ColumnIndex: i, Datatype: fieldSQLDataType(pilosa.FieldToFieldInfo(fld)), @@ -424,7 +464,7 @@ func (p *ExecutionPlanner) analyzeSource(source parser.Source, scope parser.Stat source.OutputColumns = append(source.OutputColumns, soc) } - return nil + return source, nil case *parser.TableValuedFunction: // check it actually is a table valued function - we only support one right now; subtable() @@ -432,12 +472,12 @@ func (p *ExecutionPlanner) analyzeSource(source parser.Source, scope parser.Stat case "SUBTABLE": _, err := p.analyzeCallExpression(source.Call, scope) if err != nil { - return err + return nil, err } tvfResultType, ok := source.Call.ResultDataType.(*parser.DataTypeSubtable) if !ok { - return sql3.NewErrInternalf("unexepected tvf return type") + return nil, sql3.NewErrInternalf("unexepected tvf return type") } // populate the output columns from the source @@ -452,29 +492,34 @@ func (p *ExecutionPlanner) analyzeSource(source parser.Source, scope parser.Stat } default: - return sql3.NewErrInternalf("table valued function expected") + return nil, sql3.NewErrInternalf("table valued function expected") } - return nil + return source, nil case *parser.SelectStatement: - _, err := p.analyzeSelectStatement(source) + expr, err := p.analyzeSelectStatement(source) if err != nil { - return err + return nil, err } - return nil + selExpr, ok := expr.(*parser.SelectStatement) + if !ok { + return nil, sql3.NewErrInternalf("unexpected analyzed type") + } + return selExpr, nil default: - return sql3.NewErrInternalf("unexpected source type: %T", source) + return nil, sql3.NewErrInternalf("unexpected source type: %T", source) } } func (p *ExecutionPlanner) analyzeSelectStatement(stmt *parser.SelectStatement) (parser.Expr, error) { // analyze source first - needed for name resolution - err := p.analyzeSource(stmt.Source, stmt) + source, err := p.analyzeSource(stmt.Source, stmt) if err != nil { return nil, err } + stmt.Source = source if err := p.analyzeSelectStatementWildcards(stmt); err != nil { return nil, err @@ -617,7 +662,17 @@ func (p *ExecutionPlanner) columnsFromSource(source parser.Source) ([]*parser.Re return result, nil case *parser.SelectStatement: - return nil, sql3.NewErrInternal("sub-selects are not currently supported") + for _, oc := range src.PossibleOutputColumns() { + result = append(result, &parser.ResultColumn{ + Expr: &parser.QualifiedRef{ + Table: &parser.Ident{Name: oc.TableName}, + Column: &parser.Ident{Name: oc.ColumnName}, + ColumnIndex: oc.ColumnIndex, + }, + }) + } + return result, nil + default: return nil, sql3.NewErrInternalf("unexpected source type: %T", source) } diff --git a/sql3/planner/compileshow.go b/sql3/planner/compileshow.go index 6b9fed8c8..d30a302f3 100644 --- a/sql3/planner/compileshow.go +++ b/sql3/planner/compileshow.go @@ -84,7 +84,7 @@ func (p *ExecutionPlanner) compileShowColumnsStatement(stmt *parser.ShowColumnsS tname := dax.TableName(tableName) tbl, err := p.schemaAPI.TableByName(context.Background(), tname) if err != nil { - if errors.Is(err, pilosa.ErrIndexNotFound) { + if isTableNotFoundError(err) { return nil, sql3.NewErrTableNotFound(stmt.TableName.NamePos.Line, stmt.TableName.NamePos.Column, tableName) } return nil, err @@ -174,7 +174,7 @@ func (p *ExecutionPlanner) compileShowCreateTableStatement(stmt *parser.ShowCrea tableName := parser.IdentName(stmt.TableName) tname := dax.TableName(tableName) if _, err := p.schemaAPI.TableByName(context.Background(), tname); err != nil { - if errors.Is(err, pilosa.ErrIndexNotFound) { + if isTableNotFoundError(err) { return nil, sql3.NewErrTableNotFound(stmt.TableName.NamePos.Line, stmt.TableName.NamePos.Column, tableName) } return nil, err diff --git a/sql3/planner/executionplanner.go b/sql3/planner/executionplanner.go index dbec61628..265203d5e 100644 --- a/sql3/planner/executionplanner.go +++ b/sql3/planner/executionplanner.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "strconv" + "strings" pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/logger" @@ -17,6 +18,11 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3/planner/types" ) +func isTableNotFoundError(err error) bool { + // TODO (pok) take out the second part of this check once we return correct error types across network boundaries + return errors.Is(err, dax.ErrTableNameDoesNotExist) || strings.Contains(err.Error(), "does not exist") +} + // ExecutionPlanner compiles SQL text into a query plan type ExecutionPlanner struct { executor pilosa.Executor @@ -64,10 +70,16 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen rootOperator, err = p.compileShowCreateTableStatement(stmt) case *parser.CreateTableStatement: rootOperator, err = p.compileCreateTableStatement(stmt) + case *parser.CreateViewStatement: + rootOperator, err = p.compileCreateViewStatement(stmt) case *parser.AlterTableStatement: rootOperator, err = p.compileAlterTableStatement(stmt) + case *parser.AlterViewStatement: + rootOperator, err = p.compileAlterViewStatement(stmt) case *parser.DropTableStatement: rootOperator, err = p.compileDropTableStatement(stmt) + case *parser.DropViewStatement: + rootOperator, err = p.compileDropViewStatement(stmt) case *parser.InsertStatement: rootOperator, err = p.compileInsertStatement(stmt) case *parser.BulkInsertStatement: @@ -111,10 +123,16 @@ func (p *ExecutionPlanner) analyzePlan(stmt parser.Statement) error { return nil case *parser.CreateTableStatement: return p.analyzeCreateTableStatement(stmt) + case *parser.CreateViewStatement: + return p.analyzeCreateViewStatement(stmt) case *parser.AlterTableStatement: return p.analyzeAlterTableStatement(stmt) + case *parser.AlterViewStatement: + return p.analyzeAlterViewStatement(stmt) case *parser.DropTableStatement: return nil + case *parser.DropViewStatement: + return nil case *parser.InsertStatement: return p.analyzeInsertStatement(stmt) case *parser.BulkInsertStatement: diff --git a/sql3/planner/executionplannersystemtables.go b/sql3/planner/executionplannersystemtables.go index a4a5843bc..039da5c3a 100644 --- a/sql3/planner/executionplannersystemtables.go +++ b/sql3/planner/executionplannersystemtables.go @@ -28,10 +28,10 @@ func newSystemTableDefintionsWrapper(api pilosa.SchemaAPI) *systemTableDefintion func (s *systemTableDefintionsWrapper) TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error) { tbl, err := s.schemaAPI.TableByName(ctx, tname) if err != nil { - if errors.Is(err, pilosa.ErrIndexNotFound) { + if isTableNotFoundError(err) { st, ok := systemTables[string(tname)] if !ok { - return nil, pilosa.ErrIndexNotFound + return nil, dax.NewErrTableNameDoesNotExist(tname) } return indexInfoFromSystemTableB(st) diff --git a/sql3/planner/opalterview.go b/sql3/planner/opalterview.go new file mode 100644 index 000000000..70789acb1 --- /dev/null +++ b/sql3/planner/opalterview.go @@ -0,0 +1,97 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "fmt" + + "github.com/molecula/featurebase/v3/sql3" + "github.com/molecula/featurebase/v3/sql3/planner/types" +) + +// PlanOpAlterView implements the ALTER VIEW operator +type PlanOpAlterView struct { + planner *ExecutionPlanner + view *viewSystemObject + warnings []string +} + +func NewPlanOpAlterView(planner *ExecutionPlanner, view *viewSystemObject) *PlanOpAlterView { + return &PlanOpAlterView{ + planner: planner, + view: view, + warnings: make([]string, 0), + } +} + +func (p *PlanOpAlterView) Schema() types.Schema { + return types.Schema{} +} + +func (p *PlanOpAlterView) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + return newAlterViewIter(p.planner, p.view), nil +} + +func (p *PlanOpAlterView) Children() []types.PlanOperator { + return []types.PlanOperator{} +} + +func (p *PlanOpAlterView) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + if len(children) != 0 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return NewPlanOpAlterView(p.planner, p.view), nil +} + +func (p *PlanOpAlterView) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + result["_schema"] = p.Schema().Plan() + result["model"] = p.view.name + return result +} + +func (p *PlanOpAlterView) String() string { + return "" +} + +func (p *PlanOpAlterView) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpAlterView) Warnings() []string { + var w []string + w = append(w, p.warnings...) + return w +} + +type alterViewIter struct { + planner *ExecutionPlanner + view *viewSystemObject +} + +func newAlterViewIter(planner *ExecutionPlanner, view *viewSystemObject) *alterViewIter { + return &alterViewIter{ + planner: planner, + view: view, + } +} + +func (i *alterViewIter) Next(ctx context.Context) (types.Row, error) { + // now check in the views table to see if it exists + v, err := i.planner.getViewByName(i.view.name) + if err != nil { + return nil, err + } + if v == nil { + return nil, sql3.NewErrViewNotFound(0, 0, i.view.name) + } + + // now store the view into fb_views + err = i.planner.updateView(i.view) + if err != nil { + return nil, err + } + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/opcreateview.go b/sql3/planner/opcreateview.go new file mode 100644 index 000000000..310a6bdfa --- /dev/null +++ b/sql3/planner/opcreateview.go @@ -0,0 +1,120 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "fmt" + + "github.com/molecula/featurebase/v3/dax" + "github.com/molecula/featurebase/v3/sql3" + "github.com/molecula/featurebase/v3/sql3/planner/types" +) + +// PlanOpCreateView implements the CREATE VIEW operator +type PlanOpCreateView struct { + planner *ExecutionPlanner + view *viewSystemObject + ifNotExists bool + warnings []string +} + +func NewPlanOpCreateView(planner *ExecutionPlanner, ifNotExists bool, view *viewSystemObject) *PlanOpCreateView { + return &PlanOpCreateView{ + planner: planner, + view: view, + ifNotExists: ifNotExists, + warnings: make([]string, 0), + } +} + +func (p *PlanOpCreateView) Schema() types.Schema { + return types.Schema{} +} + +func (p *PlanOpCreateView) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + return newCreateViewIter(p.planner, p.ifNotExists, p.view), nil +} + +func (p *PlanOpCreateView) Children() []types.PlanOperator { + return []types.PlanOperator{} +} + +func (p *PlanOpCreateView) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + if len(children) != 0 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return NewPlanOpCreateView(p.planner, p.ifNotExists, p.view), nil +} + +func (p *PlanOpCreateView) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + result["_schema"] = p.Schema().Plan() + result["model"] = p.view.name + return result +} + +func (p *PlanOpCreateView) String() string { + return "" +} + +func (p *PlanOpCreateView) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpCreateView) Warnings() []string { + var w []string + w = append(w, p.warnings...) + return w +} + +type createViewIter struct { + planner *ExecutionPlanner + view *viewSystemObject + ifNotExists bool +} + +func newCreateViewIter(planner *ExecutionPlanner, ifNotExists bool, view *viewSystemObject) *createViewIter { + return &createViewIter{ + planner: planner, + view: view, + ifNotExists: ifNotExists, + } +} + +func (i *createViewIter) Next(ctx context.Context) (types.Row, error) { + // make sure we have no existing table named the same as our view + viewName := dax.TableName(i.view.name) + tbl, err := i.planner.schemaAPI.TableByName(context.Background(), viewName) + if err != nil { + if !isTableNotFoundError(err) { + return nil, err + } + } + if tbl != nil { + if i.ifNotExists { + return nil, types.ErrNoMoreRows + } + return nil, sql3.NewErrViewExists(0, 0, i.view.name) + } + + // now check in the views table to see if it is exists + v, err := i.planner.getViewByName(i.view.name) + if err != nil { + return nil, err + } + if v != nil { + if i.ifNotExists { + return nil, types.ErrNoMoreRows + } + return nil, sql3.NewErrViewExists(0, 0, i.view.name) + } + + // now store the view into fb_views + err = i.planner.insertView(i.view) + if err != nil { + return nil, err + } + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/opdropview.go b/sql3/planner/opdropview.go new file mode 100644 index 000000000..c90b50469 --- /dev/null +++ b/sql3/planner/opdropview.go @@ -0,0 +1,102 @@ +// Copyright 2023 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "fmt" + + "github.com/molecula/featurebase/v3/sql3" + "github.com/molecula/featurebase/v3/sql3/planner/types" +) + +// PlanOpDropView plan operator to drop a view. +type PlanOpDropView struct { + planner *ExecutionPlanner + viewName string + ifExists bool + warnings []string +} + +func NewPlanOpDropView(p *ExecutionPlanner, ifExists bool, viewName string) *PlanOpDropView { + return &PlanOpDropView{ + planner: p, + viewName: viewName, + ifExists: ifExists, + warnings: make([]string, 0), + } +} + +func (p *PlanOpDropView) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + result["viewName"] = p.viewName + result["isExists"] = p.ifExists + return result +} + +func (p *PlanOpDropView) String() string { + return "" +} + +func (p *PlanOpDropView) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpDropView) Warnings() []string { + return p.warnings +} + +func (p *PlanOpDropView) Schema() types.Schema { + return types.Schema{} +} + +func (p *PlanOpDropView) Children() []types.PlanOperator { + return []types.PlanOperator{} +} + +func (p *PlanOpDropView) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + return &dropViewRowIter{ + planner: p.planner, + ifExists: p.ifExists, + viewName: p.viewName, + }, nil +} + +func (p *PlanOpDropView) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + return nil, nil +} + +type dropViewRowIter struct { + planner *ExecutionPlanner + ifExists bool + viewName string +} + +var _ types.RowIterator = (*dropViewRowIter)(nil) + +func (i *dropViewRowIter) Next(ctx context.Context) (types.Row, error) { + err := i.planner.checkAccess(ctx, i.viewName, accessTypeDropObject) + if err != nil { + return nil, err + } + + // check in the views table to see if it exists + v, err := i.planner.getViewByName(i.viewName) + if err != nil { + return nil, err + } + if v == nil { + if i.ifExists { + return nil, types.ErrNoMoreRows + } + return nil, sql3.NewErrViewNotFound(0, 0, i.viewName) + } + + err = i.planner.deleteView(i.viewName) + if err != nil { + return nil, err + } + + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/oppqldistinctscan.go b/sql3/planner/oppqldistinctscan.go index ef4299cb4..3ff84d0c6 100644 --- a/sql3/planner/oppqldistinctscan.go +++ b/sql3/planner/oppqldistinctscan.go @@ -142,7 +142,7 @@ func (i *distinctScanRowIter) Next(ctx context.Context) (types.Row, error) { tname := dax.TableName(i.tableName) table, err := i.planner.schemaAPI.TableByName(context.Background(), tname) if err != nil { - if errors.Is(err, pilosa.ErrIndexNotFound) { + if isTableNotFoundError(err) { return nil, sql3.NewErrInternalf("table not found '%s'", i.tableName) } return nil, err diff --git a/sql3/planner/oppqltablescan.go b/sql3/planner/oppqltablescan.go index 542b513a2..a56261871 100644 --- a/sql3/planner/oppqltablescan.go +++ b/sql3/planner/oppqltablescan.go @@ -146,7 +146,7 @@ func (i *tableScanRowIter) Next(ctx context.Context) (types.Row, error) { tname := dax.TableName(i.tableName) table, err := i.planner.schemaAPI.TableByName(context.Background(), tname) if err != nil { - if errors.Is(err, pilosa.ErrIndexNotFound) { + if isTableNotFoundError(err) { return nil, sql3.NewErrInternalf("table not found '%s'", i.tableName) } return nil, err diff --git a/sql3/planner/planoptimizer.go b/sql3/planner/planoptimizer.go index 7b8d44b3b..9368515d7 100644 --- a/sql3/planner/planoptimizer.go +++ b/sql3/planner/planoptimizer.go @@ -18,6 +18,8 @@ import ( //TODO(pok) you can't group by _id in PQL, so we need to not use a PQL group by operator here //TODO(pok) move constant folding to in here +//TODO(pok) push down filters thru subqueries with aliases + // a function prototype for all optimizer rules type OptimizerFunc func(context.Context, *ExecutionPlanner, types.PlanOperator, *OptimizerScope) (types.PlanOperator, bool, error) diff --git a/sql3/planner/systemobjects.go b/sql3/planner/systemobjects.go new file mode 100644 index 000000000..d806d7359 --- /dev/null +++ b/sql3/planner/systemobjects.go @@ -0,0 +1,247 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "time" + + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/dax" + "github.com/molecula/featurebase/v3/sql3" + "github.com/molecula/featurebase/v3/sql3/parser" + "github.com/molecula/featurebase/v3/sql3/planner/types" +) + +type viewSystemObject struct { + name string + statement string +} + +func (p *ExecutionPlanner) ensureViewsSystemTableExists() error { + _, err := p.schemaAPI.TableByName(context.Background(), "fb_views") + if err != nil { + if !isTableNotFoundError(err) { + return err + } + + // create table fb_views ( + // _id string + // name string + // statement string + // owner string + // updated_by string + // created_at timestamp + // updated_at timestamp + // ); + + // if it doesn't, create it by making the appropriate iterator + iter := &createTableRowIter{ + planner: p, + tableName: "fb_views", + failIfExists: false, + isKeyed: true, + keyPartitions: 0, + columns: []*createTableField{ + { + planner: p, + name: "name", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "statement", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "owner", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "updated_by", + typeName: dax.BaseTypeString, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeMutex(pilosa.DefaultCacheType, pilosa.DefaultCacheSize), + pilosa.OptFieldKeys(), + }, + }, + { + planner: p, + name: "created_at", + typeName: dax.BaseTypeTimestamp, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds), + }, + }, + { + planner: p, + name: "updated_at", + typeName: dax.BaseTypeTimestamp, + fos: []pilosa.FieldOption{ + pilosa.OptFieldTypeTimestamp(pilosa.DefaultEpoch, pilosa.TimeUnitSeconds), + }, + }, + }, + description: "system table for views", + } + // call next on our iterator to create the table + _, err := iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + } + return nil +} + +func (p *ExecutionPlanner) getViewByName(name string) (*viewSystemObject, error) { + err := p.ensureViewsSystemTableExists() + if err != nil { + return nil, err + } + + tbl, err := p.schemaAPI.TableByName(context.Background(), "fb_views") + if err != nil { + return nil, sql3.NewErrTableNotFound(0, 0, "fb_views") + } + + cols := make([]string, len(tbl.Fields)) + for i, c := range tbl.Fields { + cols[i] = string(c.Name) + } + + iter := &tableScanRowIter{ + planner: p, + tableName: "fb_views", + columns: cols, + predicate: newBinOpPlanExpression( + newQualifiedRefPlanExpression("fb_views", "_id", 0, parser.NewDataTypeString()), + parser.EQ, + newStringLiteralPlanExpression(name), + parser.NewDataTypeBool(), + ), + topExpr: nil, + } + + row, err := iter.Next(context.Background()) + if err != nil { + if err == types.ErrNoMoreRows { + // view does not exist + return nil, nil + } + return nil, err + } + + return &viewSystemObject{ + name: row[1].(string), + statement: row[2].(string), + }, nil +} + +func (p *ExecutionPlanner) insertView(view *viewSystemObject) error { + err := p.ensureViewsSystemTableExists() + if err != nil { + return err + } + + createTime := time.Now().UTC() + + iter := &insertRowIter{ + planner: p, + tableName: "fb_views", + targetColumns: []*qualifiedRefPlanExpression{ + newQualifiedRefPlanExpression("fb_views", "_id", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_views", "name", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_views", "statement", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_views", "owner", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_views", "updated_by", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_views", "created_at", 0, parser.NewDataTypeTimestamp()), + newQualifiedRefPlanExpression("fb_views", "updated_at", 0, parser.NewDataTypeTimestamp()), + }, + insertValues: [][]types.PlanExpression{ + { + newStringLiteralPlanExpression(view.name), + newStringLiteralPlanExpression(view.name), + newStringLiteralPlanExpression(view.statement), + newStringLiteralPlanExpression(""), + newStringLiteralPlanExpression(""), + newDateLiteralPlanExpression(createTime), + newDateLiteralPlanExpression(createTime), + }, + }, + } + _, err = iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + return nil +} + +func (p *ExecutionPlanner) updateView(view *viewSystemObject) error { + err := p.ensureViewsSystemTableExists() + if err != nil { + return err + } + + updateTime := time.Now().UTC() + + iter := &insertRowIter{ + planner: p, + tableName: "fb_views", + targetColumns: []*qualifiedRefPlanExpression{ + newQualifiedRefPlanExpression("fb_views", "_id", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_views", "statement", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_views", "updated_by", 0, parser.NewDataTypeString()), + newQualifiedRefPlanExpression("fb_views", "updated_at", 0, parser.NewDataTypeTimestamp()), + }, + insertValues: [][]types.PlanExpression{ + { + newStringLiteralPlanExpression(view.name), + newStringLiteralPlanExpression(view.statement), + newStringLiteralPlanExpression(""), + newDateLiteralPlanExpression(updateTime), + }, + }, + } + _, err = iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + return nil +} + +func (p *ExecutionPlanner) deleteView(viewName string) error { + err := p.ensureViewsSystemTableExists() + if err != nil { + return err + } + + iter := &filteredDeleteRowIter{ + planner: p, + tableName: "fb_views", + filter: newBinOpPlanExpression( + newQualifiedRefPlanExpression("fb_views", "_id", 0, parser.NewDataTypeString()), + parser.EQ, + newStringLiteralPlanExpression(viewName), + parser.NewDataTypeBool(), + ), + } + _, err = iter.Next(context.Background()) + if err != nil && err != types.ErrNoMoreRows { + return err + } + return nil +} diff --git a/sql3/sql_complex_test.go b/sql3/sql_complex_test.go index 55c0a26c3..d780314ba 100644 --- a/sql3/sql_complex_test.go +++ b/sql3/sql_complex_test.go @@ -193,7 +193,10 @@ func TestPlanner_Show(t *testing.T) { if err != nil { t.Fatal(err) } - if len(results) != 7 { + + // we keep adding system tables on the fly so as + // long as we get more than 0 tables, we're good + if len(results) == 0 { t.Fatal(fmt.Errorf("unexpected result set length")) } diff --git a/sql3/test/defs/defs.go b/sql3/test/defs/defs.go index e25e14510..42dcd6494 100644 --- a/sql3/test/defs/defs.go +++ b/sql3/test/defs/defs.go @@ -20,6 +20,7 @@ var TableTests []TableTest = []TableTest{ distinctTests, subqueryTests, + viewTests, topTests, diff --git a/sql3/test/defs/defs_views.go b/sql3/test/defs/defs_views.go new file mode 100644 index 000000000..bd894bbb6 --- /dev/null +++ b/sql3/test/defs/defs_views.go @@ -0,0 +1,126 @@ +package defs + +var viewTests = TableTest{ + name: "viewtests", + Table: tbl( + "viewtable", + srcHdrs( + srcHdr("_id", fldTypeID), + srcHdr("a_string", fldTypeString), + srcHdr("a_int", fldTypeInt), + ), + srcRows( + srcRow(int64(1), "str1", int64(10)), + srcRow(int64(2), "str1", int64(20)), + srcRow(int64(3), "str2", int64(30)), + srcRow(int64(4), "str2", int64(40)), + srcRow(int64(5), "str3", int64(50)), + ), + ), + SQLTests: []SQLTest{ + { + name: "create-view", + SQLs: sqls( + "create view viewonviewtable as select _id, a_string, a_int from viewtable;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + SortStringKeys: true, + }, + { + name: "create-view-should-fail", + SQLs: sqls( + "create view viewonviewtable as select _id, a_string, a_int from viewtable;", + ), + ExpErr: "view 'viewonviewtable' already exists", + }, + { + name: "create-view-should-not-fail", + SQLs: sqls( + "create view if not exists viewonviewtable as select _id, a_string, a_int from viewtable;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + SortStringKeys: true, + }, + { + name: "select-view", + SQLs: sqls( + "select * from viewonviewtable;", + "select _id, a_string, a_int from viewonviewtable;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + hdr("a_string", fldTypeString), + hdr("a_int", fldTypeInt), + ), + ExpRows: rows( + row(int64(1), "str1", int64(10)), + row(int64(2), "str1", int64(20)), + row(int64(3), "str2", int64(30)), + row(int64(4), "str2", int64(40)), + row(int64(5), "str3", int64(50)), + ), + Compare: CompareExactUnordered, + SortStringKeys: true, + }, + { + name: "alter-view", + SQLs: sqls( + "alter view viewonviewtable as select _id, a_string, a_int from viewtable where a_int > 20;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + SortStringKeys: true, + }, + { + name: "select-alter-view", + SQLs: sqls( + "select * from viewonviewtable;", + "select _id, a_string, a_int from viewonviewtable;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + hdr("a_string", fldTypeString), + hdr("a_int", fldTypeInt), + ), + ExpRows: rows( + row(int64(3), "str2", int64(30)), + row(int64(4), "str2", int64(40)), + row(int64(5), "str3", int64(50)), + ), + Compare: CompareExactUnordered, + SortStringKeys: true, + }, + { + name: "drop-view", + SQLs: sqls( + "drop view viewonviewtable;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + SortStringKeys: true, + }, + { + name: "drop-view-if-exists-after-drop", + SQLs: sqls( + "drop view if exists viewonviewtable;", + ), + ExpHdrs: hdrs(), + ExpRows: rows(), + Compare: CompareExactUnordered, + SortStringKeys: true, + }, + { + name: "select-view-after-drop", + SQLs: sqls( + "select * from viewonviewtable;", + ), + ExpErr: "table or view 'viewonviewtable' not found", + }, + }, +} From a9b3fd2c4d8d21fbc88c1992452871f1879d2f7c Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Tue, 17 Jan 2023 11:57:58 -0600 Subject: [PATCH 05/15] Database isolation: Balancer (#2407) * Database isolation: Balancer Remove naive Balancer remove debugging lines Thread dax.Transaction through Controller Change role to roleType Swap out Balancer interface with new one Standardize InvalidTransaction error Add some interface comments * Remove type.Worker; replace with type.Address * Remove database validate from Queryer This is already being handled in the `CreateTable()` method. Prior to doing that validation, we were getting a panic, but that's no longer the case. * Remove dax.TableQualifier; replace with dax.QualifiedDatabaseID * Update IDK test to create database (cherry picked from commit d971cfc2698c0dcd236ed4d636494a46cce9fae6) --- api.go | 54 + api_directive_test.go | 8 +- dax/address.go | 7 + dax/boltdb/boltdb.go | 7 + dax/boltdb/directiveversion.go | 16 +- dax/boltdb/node.go | 51 +- dax/boltdb/node_test.go | 14 +- dax/directive.go | 6 +- dax/errors.go | 26 + dax/mds/client/client.go | 72 +- dax/mds/controller/balancer.go | 75 +- dax/mds/controller/balancer/balancer.go | 821 ++++++++++++ dax/mds/controller/balancer/balancer_test.go | 1104 ++++++++++++++++ .../controller/balancer/boltdb/balancer.go | 813 ++++++++++++ .../controller/{naive => balancer}/types.go | 26 +- dax/mds/controller/config.go | 8 +- dax/mds/controller/controller.go | 1125 ++++++++++------- dax/mds/controller/controller_test.go | 581 +++++---- dax/mds/controller/errors.go | 2 +- dax/mds/controller/naive/balancer.go | 599 --------- dax/mds/controller/naive/balancer_test.go | 751 ----------- dax/mds/controller/naive/boltdb/balancer.go | 606 --------- .../controller/naive/boltdb/balancer_test.go | 721 ----------- dax/mds/controller/sets.go | 6 +- dax/mds/controller/snapping_turtle.go | 43 +- dax/mds/controller/stringers.go | 8 + dax/mds/http/handler.go | 63 +- dax/mds/mds.go | 245 +--- dax/mds/schemar/boltdb/schemar.go | 392 ++++-- dax/mds/schemar/boltdb/schemar_test.go | 129 +- dax/mds/schemar/errors.go | 17 + dax/mds/schemar/http/handler.go | 199 --- dax/mds/schemar/schemar.go | 82 +- dax/mds/schemar/schemar_test.go | 124 -- dax/node.go | 27 +- dax/queryer/client/client.go | 13 +- dax/queryer/http/handler.go | 10 +- dax/queryer/orchestrator.go | 9 +- dax/queryer/queryer.go | 92 +- dax/queryer/schema_api.go | 34 +- dax/role.go | 8 +- dax/schema.go | 58 +- dax/storage/storage_test.go | 8 +- dax/table.go | 309 +++-- dax/table_test.go | 12 +- dax/test/dax/dax_test.go | 133 +- dax/test/table.go | 28 +- dax/transaction.go | 9 + dax/workerjob.go | 38 +- idk/ingest.go | 8 +- idk/ingest_test.go | 25 +- idk/mds/importer.go | 14 +- idk/mds/schemamanager.go | 8 +- sql3/planner/compilecreateview.go | 4 +- sql3/planner/compiledropview.go | 6 +- sql3/planner/opalterview.go | 4 +- sql3/planner/opcreateview.go | 6 +- sql3/planner/opdropview.go | 4 +- sql3/planner/opfeaturebasetables.go | 14 +- sql3/planner/systemobjects.go | 10 +- 60 files changed, 5183 insertions(+), 4509 deletions(-) create mode 100644 dax/mds/controller/balancer/balancer.go create mode 100644 dax/mds/controller/balancer/balancer_test.go create mode 100644 dax/mds/controller/balancer/boltdb/balancer.go rename dax/mds/controller/{naive => balancer}/types.go (74%) delete mode 100644 dax/mds/controller/naive/balancer.go delete mode 100644 dax/mds/controller/naive/balancer_test.go delete mode 100644 dax/mds/controller/naive/boltdb/balancer.go delete mode 100644 dax/mds/controller/naive/boltdb/balancer_test.go delete mode 100644 dax/mds/schemar/http/handler.go delete mode 100644 dax/mds/schemar/schemar_test.go create mode 100644 dax/transaction.go diff --git a/api.go b/api.go index ae61eb99a..5e7bfa418 100644 --- a/api.go +++ b/api.go @@ -3354,6 +3354,9 @@ type QueryAPI interface { Query(ctx context.Context, req *QueryRequest) (QueryResponse, error) } +// Ensure type implements interface. +var _ SystemAPI = (*FeatureBaseSystemAPI)(nil) + // FeatureBaseSystemAPI is a wrapper around pilosa.API. It implements the // SystemAPI interface type FeatureBaseSystemAPI struct { @@ -3432,3 +3435,54 @@ func (fsapi *FeatureBaseSystemAPI) ClusterNodes() []ClusterNode { return result } + +// Ensure type implements interface. +var _ SystemAPI = (*NopSystemAPI)(nil) + +// NopSystemAPI is a no-op implementation of the SystemAPI. +type NopSystemAPI struct{} + +func (napi *NopSystemAPI) ClusterName() string { + return "" +} + +func (napi *NopSystemAPI) Version() string { + return "" +} + +func (napi *NopSystemAPI) PlatformDescription() string { + return "" +} + +func (napi *NopSystemAPI) PlatformVersion() string { + return "" +} + +func (napi *NopSystemAPI) ClusterNodeCount() int { + return 0 +} + +func (napi *NopSystemAPI) ClusterReplicaCount() int { + return 0 +} + +func (napi *NopSystemAPI) ShardWidth() int { + return 0 +} + +func (napi *NopSystemAPI) ClusterState() string { + return "" +} + +func (napi *NopSystemAPI) DataDir() string { + return "" +} + +func (napi *NopSystemAPI) NodeID() string { + return "" +} + +func (napi *NopSystemAPI) ClusterNodes() []ClusterNode { + result := make([]ClusterNode, 0) + return result +} diff --git a/api_directive_test.go b/api_directive_test.go index fffa1ce77..a90a3b1ba 100644 --- a/api_directive_test.go +++ b/api_directive_test.go @@ -20,10 +20,10 @@ func TestAPI_Directive(t *testing.T) { api := c.GetPrimary().API ctx := context.Background() - qual := dax.NewTableQualifier("acme", "db1") - tbl1 := daxtest.TestQualifiedTableWithID(t, qual, "1", "tbl1", 12, false) - tbl2 := daxtest.TestQualifiedTableWithID(t, qual, "2", "tbl2", 12, false) - tbl3 := daxtest.TestQualifiedTableWithID(t, qual, "3", "tbl3", 12, false) + qdbid := dax.NewQualifiedDatabaseID("acme", "db1") + tbl1 := daxtest.TestQualifiedTableWithID(t, qdbid, "1", "tbl1", 12, false) + tbl2 := daxtest.TestQualifiedTableWithID(t, qdbid, "2", "tbl2", 12, false) + tbl3 := daxtest.TestQualifiedTableWithID(t, qdbid, "3", "tbl3", 12, false) t.Run("Schema", func(t *testing.T) { diff --git a/dax/address.go b/dax/address.go index 99e65ff7d..ab5e45ec6 100644 --- a/dax/address.go +++ b/dax/address.go @@ -148,6 +148,13 @@ func (a addr) hostPortPath() string { return ret } +// Addresses is a sortable slice of Address. +type Addresses []Address + +func (a Addresses) Len() int { return len(a) } +func (a Addresses) Less(i, j int) bool { return a[i] < a[j] } +func (a Addresses) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + // AddressManager is an interface for any service which needs to maintain a list // of addresses, and receive add/remove address requests from other services. type AddressManager interface { diff --git a/dax/boltdb/boltdb.go b/dax/boltdb/boltdb.go index 1ddaf795c..aa50bd27d 100644 --- a/dax/boltdb/boltdb.go +++ b/dax/boltdb/boltdb.go @@ -125,6 +125,7 @@ func (db *DB) Close() (err error) { // BeginTx starts a transaction and returns a wrapper Tx type. This type // provides a reference to the database and a fixed timestamp at the start of // the transaction. The timestamp allows us to mock time during tests as well. +// The wrapper also contains the context. func (db *DB) BeginTx(ctx context.Context, writable bool) (*Tx, error) { tx, err := db.db.Begin(writable) if err != nil { @@ -134,6 +135,7 @@ func (db *DB) BeginTx(ctx context.Context, writable bool) (*Tx, error) { // Return wrapper Tx that includes the transaction start time. return &Tx{ Tx: tx, + ctx: ctx, db: db, now: db.Now().UTC().Truncate(time.Second), }, nil @@ -142,10 +144,15 @@ func (db *DB) BeginTx(ctx context.Context, writable bool) (*Tx, error) { // Tx wraps the SQL Tx object to provide a timestamp at the start of the transaction. type Tx struct { *bolt.Tx + ctx context.Context db *DB now time.Time } +func (tx *Tx) Context() context.Context { + return tx.ctx +} + func (db *DB) Path() string { return db.filePath } diff --git a/dax/boltdb/directiveversion.go b/dax/boltdb/directiveversion.go index b00452c6a..c4c04fe44 100644 --- a/dax/boltdb/directiveversion.go +++ b/dax/boltdb/directiveversion.go @@ -1,7 +1,6 @@ package boltdb import ( - "context" "encoding/binary" "github.com/featurebasedb/featurebase/v3/dax" @@ -32,14 +31,13 @@ func NewDirectiveVersion(db *DB) *DirectiveVersion { } } -func (d *DirectiveVersion) Increment(ctx context.Context, delta uint64) (uint64, error) { - tx, err := d.db.BeginTx(ctx, true) - if err != nil { - return 0, errors.Wrap(err, "getting transaction") +func (d *DirectiveVersion) Increment(tx dax.Transaction, delta uint64) (uint64, error) { + txx, ok := tx.(*Tx) + if !ok { + return 0, dax.NewErrInvalidTransaction() } - defer tx.Rollback() - bkt := tx.Bucket(bucketDirective) + bkt := txx.Bucket(bucketDirective) if bkt == nil { return 0, errors.Errorf(ErrFmtBucketNotFound, bucketDirective) } @@ -58,9 +56,5 @@ func (d *DirectiveVersion) Increment(ctx context.Context, delta uint64) (uint64, return 0, errors.Wrap(err, "putting next directive version") } - if err := tx.Commit(); err != nil { - return 0, err - } - return nextVersion, nil } diff --git a/dax/boltdb/node.go b/dax/boltdb/node.go index 583ebb01d..b06b3912d 100644 --- a/dax/boltdb/node.go +++ b/dax/boltdb/node.go @@ -2,7 +2,6 @@ package boltdb import ( "bytes" - "context" "encoding/json" "fmt" @@ -39,14 +38,13 @@ func NewNodeService(db *DB, logger logger.Logger) *NodeService { } } -func (s *NodeService) CreateNode(ctx context.Context, addr dax.Address, node *dax.Node) error { - tx, err := s.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "getting transaction") +func (s *NodeService) CreateNode(tx dax.Transaction, addr dax.Address, node *dax.Node) error { + txx, ok := tx.(*Tx) + if !ok { + return dax.NewErrInvalidTransaction() } - defer tx.Rollback() - bkt := tx.Bucket(bucketNodes) + bkt := txx.Bucket(bucketNodes) if bkt == nil { return errors.Errorf(ErrFmtBucketNotFound, bucketNodes) } @@ -60,17 +58,16 @@ func (s *NodeService) CreateNode(ctx context.Context, addr dax.Address, node *da return errors.Wrap(err, "putting node") } - return tx.Commit() + return nil } -func (s *NodeService) ReadNode(ctx context.Context, addr dax.Address) (*dax.Node, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrap(err, "beginning tx") +func (s *NodeService) ReadNode(tx dax.Transaction, addr dax.Address) (*dax.Node, error) { + txx, ok := tx.(*Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() } - defer tx.Rollback() - bkt := tx.Bucket(bucketNodes) + bkt := txx.Bucket(bucketNodes) if bkt == nil { return nil, errors.Errorf(ErrFmtBucketNotFound, bucketNodes) } @@ -88,14 +85,13 @@ func (s *NodeService) ReadNode(ctx context.Context, addr dax.Address) (*dax.Node return node, nil } -func (s *NodeService) DeleteNode(ctx context.Context, addr dax.Address) error { - tx, err := s.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "beginning tx") +func (s *NodeService) DeleteNode(tx dax.Transaction, addr dax.Address) error { + txx, ok := tx.(*Tx) + if !ok { + return dax.NewErrInvalidTransaction() } - defer tx.Rollback() - bkt := tx.Bucket(bucketNodes) + bkt := txx.Bucket(bucketNodes) if bkt == nil { return errors.Errorf(ErrFmtBucketNotFound, bucketNodes) } @@ -104,17 +100,16 @@ func (s *NodeService) DeleteNode(ctx context.Context, addr dax.Address) error { return errors.Wrapf(err, "deleting node key: %s", addressKey(addr)) } - return tx.Commit() + return nil } -func (s *NodeService) Nodes(ctx context.Context) ([]*dax.Node, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrap(err, "getting tx") +func (s *NodeService) Nodes(tx dax.Transaction) ([]*dax.Node, error) { + txx, ok := tx.(*Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() } - defer tx.Rollback() - nodes, err := s.getNodes(ctx, tx) + nodes, err := s.getNodes(txx) if err != nil { return nil, errors.Wrap(err, "getting nodes") } @@ -122,7 +117,7 @@ func (s *NodeService) Nodes(ctx context.Context) ([]*dax.Node, error) { return nodes, nil } -func (s *NodeService) getNodes(ctx context.Context, tx *Tx) ([]*dax.Node, error) { +func (s *NodeService) getNodes(tx *Tx) ([]*dax.Node, error) { c := tx.Bucket(bucketNodes).Cursor() // Deserialize rows into Node objects. diff --git a/dax/boltdb/node_test.go b/dax/boltdb/node_test.go index f9791ae5e..591bb6968 100644 --- a/dax/boltdb/node_test.go +++ b/dax/boltdb/node_test.go @@ -35,21 +35,27 @@ func TestNodeService(t *testing.T) { }, } + tx, err := db.BeginTx(ctx, true) + assert.NoError(t, err) + defer tx.Rollback() + // Create node. - assert.NoError(t, ns.CreateNode(ctx, node1.Address, node1)) + assert.NoError(t, ns.CreateNode(tx, node1.Address, node1)) // Read node. - n, err := ns.ReadNode(ctx, node1.Address) + n, err := ns.ReadNode(tx, node1.Address) assert.NoError(t, err) assert.Equal(t, node1, n) // Delete node. - assert.NoError(t, ns.DeleteNode(ctx, node1.Address)) + assert.NoError(t, ns.DeleteNode(tx, node1.Address)) // Read node. - _, err = ns.ReadNode(ctx, node1.Address) + _, err = ns.ReadNode(tx, node1.Address) if assert.Error(t, err) { assert.True(t, errors.Is(err, dax.ErrNodeDoesNotExist)) } + + assert.NoError(t, tx.Commit()) }) } diff --git a/dax/directive.go b/dax/directive.go index a0eccad26..c7ffde19a 100644 --- a/dax/directive.go +++ b/dax/directive.go @@ -1,7 +1,5 @@ package dax -import "context" - // Directive contains the instructions, sent from MDS, which a compute node is // to follow. A Directive is typically JSON-encoded and POSTed to a compute // node's `/directive` endpoint. @@ -22,7 +20,7 @@ type Directive struct { } type DirectiveVersion interface { - Increment(ctx context.Context, delta uint64) (uint64, error) + Increment(tx Transaction, delta uint64) (uint64, error) } // DirectiveMethod is used to tell the compute node how it should handle the @@ -177,5 +175,5 @@ func (d *Directive) IsEmpty() bool { type Directives []*Directive func (d Directives) Len() int { return len(d) } -func (d Directives) Less(i, j int) bool { return d[i].Address.String() < d[j].Address.String() } +func (d Directives) Less(i, j int) bool { return d[i].Version < d[j].Version } func (d Directives) Swap(i, j int) { d[i], d[j] = d[j], d[i] } diff --git a/dax/errors.go b/dax/errors.go index d5bee2627..728b297ea 100644 --- a/dax/errors.go +++ b/dax/errors.go @@ -7,6 +7,9 @@ import ( ) const ( + ErrDatabaseIDExists errors.Code = "DatabaseIDExists" + ErrDatabaseIDDoesNotExist errors.Code = "DatabaseIDDoesNotExist" + ErrTableIDExists errors.Code = "TableIDExists" ErrTableKeyExists errors.Code = "TableKeyExists" ErrTableNameExists errors.Code = "TableNameExists" @@ -17,12 +20,28 @@ const ( ErrFieldExists errors.Code = "FieldExists" ErrFieldDoesNotExist errors.Code = "FieldDoesNotExist" + ErrInvalidTransaction errors.Code = "InvalidTransaction" + ErrUnimplemented errors.Code = "Unimplemented" ) // The following are helper functions for constructing coded errors containing // relevant information about the specific error. +func NewErrDatabaseIDExists(qdbid QualifiedDatabaseID) error { + return errors.New( + ErrDatabaseIDExists, + fmt.Sprintf("database ID '%s' already exists", qdbid), + ) +} + +func NewErrDatabaseIDDoesNotExist(qdbid QualifiedDatabaseID) error { + return errors.New( + ErrDatabaseIDDoesNotExist, + fmt.Sprintf("database ID '%s' does not exist", qdbid), + ) +} + func NewErrTableIDDoesNotExist(qtid QualifiedTableID) error { return errors.New( ErrTableIDDoesNotExist, @@ -78,3 +97,10 @@ func NewErrFieldExists(fieldName FieldName) error { fmt.Sprintf("field '%s' already exists", fieldName), ) } + +func NewErrInvalidTransaction() error { + return errors.New( + ErrInvalidTransaction, + "tx is not a *boltdb.Tx", + ) +} diff --git a/dax/mds/client/client.go b/dax/mds/client/client.go index d105d1f1a..3f2c1c163 100644 --- a/dax/mds/client/client.go +++ b/dax/mds/client/client.go @@ -54,8 +54,8 @@ func (c *Client) TableByID(ctx context.Context, qtid dax.QualifiedTableID) (*dax } // TODO(tlt): collapse TableID into this -func (c *Client) TableByName(ctx context.Context, qual dax.TableQualifier, tname dax.TableName) (*dax.QualifiedTable, error) { - qtid, err := c.TableID(ctx, qual, tname) +func (c *Client) TableByName(ctx context.Context, qdbid dax.QualifiedDatabaseID, tname dax.TableName) (*dax.QualifiedTable, error) { + qtid, err := c.TableID(ctx, qdbid, tname) if err != nil { return nil, errors.Wrap(err, "getting table id") } @@ -93,14 +93,14 @@ func (c *Client) Table(ctx context.Context, qtid dax.QualifiedTableID) (*dax.Qua return qtable, nil } -func (c *Client) TableID(ctx context.Context, qual dax.TableQualifier, name dax.TableName) (dax.QualifiedTableID, error) { +func (c *Client) TableID(ctx context.Context, qdbid dax.QualifiedDatabaseID, name dax.TableName) (dax.QualifiedTableID, error) { url := fmt.Sprintf("%s/table-id", c.address.WithScheme(defaultScheme)) dflt := dax.QualifiedTableID{} req := dax.QualifiedTableID{ - TableQualifier: qual, - Name: name, + QualifiedDatabaseID: qdbid, + Name: name, } // Encode the request. @@ -130,12 +130,12 @@ func (c *Client) TableID(ctx context.Context, qual dax.TableQualifier, name dax. return qtid, nil } -func (c *Client) Tables(ctx context.Context, qual dax.TableQualifier, ids ...dax.TableID) ([]*dax.QualifiedTable, error) { +func (c *Client) Tables(ctx context.Context, qdbid dax.QualifiedDatabaseID, ids ...dax.TableID) ([]*dax.QualifiedTable, error) { url := fmt.Sprintf("%s/tables", c.address.WithScheme(defaultScheme)) req := mdshttp.TablesRequest{ - OrganizationID: qual.OrganizationID, - DatabaseID: qual.DatabaseID, + OrganizationID: qdbid.OrganizationID, + DatabaseID: qdbid.DatabaseID, TableIDs: ids, } @@ -166,6 +166,62 @@ func (c *Client) Tables(ctx context.Context, qual dax.TableQualifier, ids ...dax return qtables, nil } +func (c *Client) CreateDatabase(ctx context.Context, qdb *dax.QualifiedDatabase) error { + url := fmt.Sprintf("%s/create-database", c.address.WithScheme(defaultScheme)) + + // Encode the request. + postBody, err := json.Marshal(qdb) + if err != nil { + return errors.Wrap(err, "marshalling post request") + } + responseBody := bytes.NewBuffer(postBody) + + // Post the request. + resp, err := http.Post(url, "application/json", responseBody) + if err != nil { + return errors.Wrap(err, "posting create database request") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return errors.Errorf("status code: %d: %s", resp.StatusCode, b) + } + + return nil +} + +func (c *Client) DatabaseByID(ctx context.Context, qdbid dax.QualifiedDatabaseID) (*dax.QualifiedDatabase, error) { + url := fmt.Sprintf("%s/database-by-id", c.address.WithScheme(defaultScheme)) + + // Encode the request. + postBody, err := json.Marshal(qdbid) + if err != nil { + return nil, errors.Wrap(err, "marshalling post request") + } + responseBody := bytes.NewBuffer(postBody) + + // Post the request. + c.logger.Debugf("POST database request: url: %s", url) + resp, err := http.Post(url, "application/json", responseBody) + if err != nil { + return nil, errors.Wrap(err, "posting table request") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, errors.Errorf("status code: %d: %s", resp.StatusCode, b) + } + + var qdb *dax.QualifiedDatabase + if err := json.NewDecoder(resp.Body).Decode(&qdb); err != nil { + return nil, errors.Wrap(err, "reading response body") + } + + return qdb, nil +} + func (c *Client) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) error { url := fmt.Sprintf("%s/create-table", c.address.WithScheme(defaultScheme)) diff --git a/dax/mds/controller/balancer.go b/dax/mds/controller/balancer.go index a6402c6fb..567c4b8c7 100644 --- a/dax/mds/controller/balancer.go +++ b/dax/mds/controller/balancer.go @@ -8,31 +8,38 @@ import ( ) type Balancer interface { - AddWorker(ctx context.Context, worker fmt.Stringer) ([]dax.WorkerDiff, error) - RemoveWorker(ctx context.Context, worker fmt.Stringer) ([]dax.WorkerDiff, error) - AddJobs(ctx context.Context, job ...fmt.Stringer) ([]dax.WorkerDiff, error) - RemoveJob(ctx context.Context, job fmt.Stringer) ([]dax.WorkerDiff, error) - Balance(ctx context.Context) ([]dax.WorkerDiff, error) - CurrentState(ctx context.Context) ([]dax.WorkerInfo, error) - WorkerState(ctx context.Context, worker dax.Worker) (dax.WorkerInfo, error) - WorkersForJobs(ctx context.Context, jobs []dax.Job) ([]dax.WorkerInfo, error) + // AddWorker adds a worker to the global pool of available workers. + AddWorker(tx dax.Transaction, node *dax.Node) ([]dax.WorkerDiff, error) - // WorkersForJobPrefix returns all workers and their job - // assignments which start with `prefix` for all jobs that start - // with `prefix`. If there are free jobs that start with `prefix` - // an error is returned. - // - // The motivating use case is getting all workers for a particular - // table so we can execute a query that will hit every shard in a - // table. If there are jobs representing shards in that table - // which are not assigned to any worker, that means the query - // would return incomplete data, so we want to error. - WorkersForJobPrefix(ctx context.Context, prefix string) ([]dax.WorkerInfo, error) + // RemoveWorker removes a worker from the system. If the worker is currently + // assigned to a database and has jobs, it will be removed and its jobs will + // be either transferred to other workers or placed on the free job list. + RemoveWorker(tx dax.Transaction, addr dax.Address) ([]dax.WorkerDiff, error) - // RemoveJobs is for e.g. when dropping a table remove all jobs - // associated with that table without needing to look up in - // advance which shards or partitions are actually present. - RemoveJobs(ctx context.Context, prefix string) ([]dax.WorkerDiff, error) + // AddJobs adds new jobs for the given database. + AddJobs(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID, jobs ...dax.Job) ([]dax.WorkerDiff, error) + + // RemoveJobs removes jobs for the given database. + RemoveJobs(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID, jobs ...dax.Job) ([]dax.WorkerDiff, error) + + // BalanceDatabase forces a database balance. TODO(tlt): currently this is + // only used in tests, so perhaps we can get rid of it. + BalanceDatabase(tx dax.Transaction, qdbid dax.QualifiedDatabaseID) ([]dax.WorkerDiff, error) + + // CurrentState returns the workers and jobs currently active for the given + // database. + CurrentState(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) ([]dax.WorkerInfo, error) + + // WorkerState returns the jobs currently active for the given worker. + WorkerState(tx dax.Transaction, roleType dax.RoleType, addr dax.Address) (dax.WorkerInfo, error) + + // WorkersForJobs returns the workers and jobs currently responsible for the + // given jobs. + WorkersForJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs ...dax.Job) ([]dax.WorkerInfo, error) + + // WorkersForTable returns the workers responsible for any job related to + // the given table. + WorkersForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) ([]dax.WorkerInfo, error) } // Ensure type implements interface. @@ -45,34 +52,30 @@ func NewNopBalancer() *NopBalancer { return &NopBalancer{} } -func (b *NopBalancer) AddWorker(ctx context.Context, worker fmt.Stringer) ([]dax.WorkerDiff, error) { +func (b *NopBalancer) AddWorker(tx dax.Transaction, node *dax.Node) ([]dax.WorkerDiff, error) { return []dax.WorkerDiff{}, nil } -func (b *NopBalancer) RemoveWorker(ctx context.Context, worker fmt.Stringer) ([]dax.WorkerDiff, error) { +func (b *NopBalancer) RemoveWorker(tx dax.Transaction, addr dax.Address) ([]dax.WorkerDiff, error) { return []dax.WorkerDiff{}, nil } -func (b *NopBalancer) AddJobs(ctx context.Context, job ...fmt.Stringer) ([]dax.WorkerDiff, error) { +func (b *NopBalancer) AddJobs(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID, jobs ...dax.Job) ([]dax.WorkerDiff, error) { return []dax.WorkerDiff{}, nil } -func (b *NopBalancer) RemoveJob(ctx context.Context, job fmt.Stringer) ([]dax.WorkerDiff, error) { +func (b *NopBalancer) RemoveJobs(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID, jobs ...dax.Job) ([]dax.WorkerDiff, error) { return []dax.WorkerDiff{}, nil } -func (b *NopBalancer) Balance(ctx context.Context) ([]dax.WorkerDiff, error) { +func (b *NopBalancer) BalanceDatabase(tx dax.Transaction, qdbid dax.QualifiedDatabaseID) ([]dax.WorkerDiff, error) { return []dax.WorkerDiff{}, nil } -func (b *NopBalancer) CurrentState(ctx context.Context) ([]dax.WorkerInfo, error) { +func (b *NopBalancer) CurrentState(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) ([]dax.WorkerInfo, error) { return []dax.WorkerInfo{}, nil } -func (b *NopBalancer) WorkerState(ctx context.Context, worker dax.Worker) (dax.WorkerInfo, error) { +func (b *NopBalancer) WorkerState(tx dax.Transaction, roleType dax.RoleType, addr dax.Address) (dax.WorkerInfo, error) { return dax.WorkerInfo{}, nil } -func (b *NopBalancer) WorkersForJobs(ctx context.Context, jobs []dax.Job) ([]dax.WorkerInfo, error) { +func (b *NopBalancer) WorkersForJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs ...dax.Job) ([]dax.WorkerInfo, error) { return []dax.WorkerInfo{}, nil } -func (b *NopBalancer) WorkersForJobPrefix(ctx context.Context, prefix string) ([]dax.WorkerInfo, error) { +func (b *NopBalancer) WorkersForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) ([]dax.WorkerInfo, error) { return []dax.WorkerInfo{}, nil } - -func (b *NopBalancer) RemoveJobs(ctx context.Context, prefix string) ([]dax.WorkerDiff, error) { - return nil, nil -} diff --git a/dax/mds/controller/balancer/balancer.go b/dax/mds/controller/balancer/balancer.go new file mode 100644 index 000000000..f29813819 --- /dev/null +++ b/dax/mds/controller/balancer/balancer.go @@ -0,0 +1,821 @@ +// Package balancer is an implementation of the controller's Balancer interface. +package balancer + +import ( + "log" + "math" + "sort" + "strings" + "time" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/mds/controller" + "github.com/featurebasedb/featurebase/v3/dax/mds/schemar" + "github.com/featurebasedb/featurebase/v3/errors" + "github.com/featurebasedb/featurebase/v3/logger" +) + +// Ensure type implements interface. +var _ controller.Balancer = (*Balancer)(nil) + +// Balancer is an implementation of the controller.Balancer interface which +// isolates workers and jobs by database. It helps manage the relationships +// between workers and jobs. The logic it uses to balance jobs across workers is +// very simple; it bases everything off the number of workers and number of +// jobs. It does not take anything else (such as job size, worker capabilities, +// etc) into consideration. +type Balancer struct { + // current represents the current state of worker/job assigments. + current WorkerJobService + + // freeJobs is the set of jobs which have yet to be assigned to a worker. + // This could be because there are no available workers, or because a worker + // has been removed and the jobs for which it was responsible have yet to be + // reassigned. + freeJobs FreeJobService + + freeWorkers FreeWorkerService + + schemar schemar.Schemar + + logger logger.Logger +} + +// New returns a new instance of Balancer. +func New(fjs FreeJobService, wjs WorkerJobService, fws FreeWorkerService, schemar schemar.Schemar, logger logger.Logger) *Balancer { + return &Balancer{ + current: wjs, + freeJobs: fjs, + freeWorkers: fws, + schemar: schemar, + logger: logger, + } +} + +// AddWorker adds the given Node to the Balancer's available worker pool. +// TODO(tlt): this method takes a Node (as opposed to a Worker) because in the +// future we may want to maintain separate worker pools based on RoleType +// (compute, translate, etc.). +func (b *Balancer) AddWorker(tx dax.Transaction, node *dax.Node) ([]dax.WorkerDiff, error) { + addr := node.Address + b.logger.Debugf("AddWorker(%s)", addr) + + diffs := NewInternalDiffs() + + // This logic means that a node is used for ALL of the role types specified. + // In other words, specifying roleTypes = {compute, translate}, does not + // mean that the node can be used as either a compute worker or a translate + // worker. It means that it will be used as both. + for _, rt := range node.RoleTypes { + if err := b.addWorker(tx, rt, addr); err != nil { + return nil, errors.Wrapf(err, "adding worker: (%s) %s", rt, addr) + } + } + + // Process the freeWorkers. + // TODO(tlt): this is a little heavy-handed. I'm sure we'll need to be more + // intentional about knowing which databases needs workers, as opposed to + // this brute force loop over all databases every time. + if diff, err := b.balance(tx); err != nil { + return nil, errors.Wrapf(err, "balancing new worker: %s", addr) + } else { + diffs.Merge(diff) + } + + return diffs.Output(), nil +} + +// addWorker adds a worker to the free worker list. From there, it can be used +// by any database which needs a worker. +func (b *Balancer) addWorker(tx dax.Transaction, roleType dax.RoleType, addr dax.Address) error { + // If this worker already exists, don't do anything. + if dbkey := b.current.DatabaseForWorker(tx, addr); dbkey != "" { + return nil + } + + if err := b.freeWorkers.AddWorkers(tx, roleType, addr); err != nil { + return errors.Wrap(err, "adding free worker") + } + + return nil +} + +func (b *Balancer) assignMinWorkers(tx dax.Transaction, roleType dax.RoleType) (InternalDiffs, error) { + // Find out how many free workers we have. + freeWorkers, err := b.freeWorkers.ListWorkers(tx, roleType) + if err != nil { + return nil, errors.Wrap(err, "getting free worker list") + } + freeWorkerCount := len(freeWorkers) + + // If there are no free workers, return early. + if freeWorkerCount == 0 { + return InternalDiffs{}, nil + } + + // Get all database and their minWorkerCount (Database.Options.WorkersMin). + qdbs, err := b.schemar.Databases(tx, "") + if err != nil { + return nil, errors.Wrap(err, "getting all database") + } + + // Create a map[database]int where int is the number of workers required to + // reach that database's minWorkerCount. This map will only contain database + // which need more workers in order to reach their minimum. + m := make(map[dax.QualifiedDatabaseID]int) + + for _, qdb := range qdbs { + qdbid := qdb.QualifiedID() + + minWorkers := qdb.Options.WorkersMin + if minWorkers == 0 { + continue + } + + // If the database doesn't have any jobs, there's no need to go any + // further. In other words, we don't want to assign a worker to a + // database until it has at least one job. + if hasJobs, err := b.databaseHasJobs(tx, roleType, qdbid); err != nil { + return nil, errors.Wrapf(err, "checking has jobs: (%s) %s", roleType, qdbid) + } else if !hasJobs { + continue + } + + // Get the number of workers assigned to this database. + workerCount, err := b.current.WorkerCount(tx, roleType, qdbid) + if err != nil { + return nil, errors.Wrapf(err, "getting worker count: (%s) %s", roleType, qdbid) + } + + diff := minWorkers - workerCount + + // If we have more workers than the min required, or if we have the + // exact number of workers , don't do anything for that database. + if diff <= 0 { + continue + } + + m[qdbid] = diff + } + + diffs := NewInternalDiffs() + + // Create an ordered slice of map keys so that tests are predicatable. + qdbids := make([]dax.QualifiedDatabaseID, 0, len(m)) + for qdbid := range m { + qdbids = append(qdbids, qdbid) + } + sort.Sort(dax.QualifiedDatabaseIDs(qdbids)) + + // For each database, if there are enough free workers to + // satisfy its min, then pop that number of workers from the free list. If + // not, contine to the next database until either reaching the end of the + // database list or until there are no more free workers in the list, + // whichever comes first. + for _, qdbid := range qdbids { + need := m[qdbid] + + if freeWorkerCount == 0 { + break + } + + if freeWorkerCount >= need { + addrs, err := b.freeWorkers.PopWorkers(tx, roleType, need) + if err != nil { + return nil, errors.Wrapf(err, "popping free worker: (%s)", roleType) + } + + if diff, err := b.addDatabaseWorkers(tx, roleType, qdbid, addrs...); err != nil { + return nil, errors.Wrapf(err, "adding database workers: (%s) %s, %v", roleType, qdbid, addrs) + } else { + diffs.Merge(diff) + } + } + } + + return diffs, nil +} + +// addDatabaseWorkers adds workers from the free worker list to the pool of +// workers for a specific database. +func (b *Balancer) addDatabaseWorkers(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addrs ...dax.Address) (InternalDiffs, error) { + for _, addr := range addrs { + if err := b.current.CreateWorker(tx, roleType, qdbid, addr); err != nil { + return nil, errors.Wrap(err, "creating worker") + } + } + + // Process the freeJobs. + return b.processFreeJobs(tx, roleType, qdbid) +} + +// databaseHasJobs returns true if the database has at least one job. +func (b *Balancer) databaseHasJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (bool, error) { + // Free jobs. + if freeJobs, err := b.freeJobs.ListJobs(tx, roleType, qdbid); err != nil { + return false, errors.Wrapf(err, "getting free jobs: (%s) %s", roleType, qdbid) + } else if len(freeJobs) > 0 { + return true, nil + } + + // Assigned jobs. + if wis, err := b.current.WorkersJobs(tx, roleType, qdbid); err != nil { + return false, errors.Wrapf(err, "getting free jobs: (%s) %s", roleType, qdbid) + } else { + for _, wi := range wis { + if len(wi.Jobs) > 0 { + return true, nil + } + } + } + + return false, nil +} + +func (b *Balancer) RemoveWorker(tx dax.Transaction, addr dax.Address) ([]dax.WorkerDiff, error) { + diffs := NewInternalDiffs() + + // See if the worker is assigned to a database. + dbkey := b.current.DatabaseForWorker(tx, addr) + if dbkey == "" { + return diffs.Output(), nil + } + qdbid := dbkey.QualifiedDatabaseID() + + for _, rt := range []dax.RoleType{dax.RoleTypeCompute, dax.RoleTypeTranslate} { + // Remove the worker form the free worker list (if it's there). + if err := b.freeWorkers.RemoveWorker(tx, rt, addr); err != nil { + return nil, errors.Wrapf(err, "removing worker from free list: (%s) %s", rt, addr) + } + + if diff, err := b.removeDatabaseWorker(tx, rt, qdbid, addr); err != nil { + return nil, errors.Wrapf(err, "removing worker: (%s) %s", rt, addr) + } else { + diffs.Merge(diff) + } + } + + // Balance the affected database. + if diff, err := b.balanceDatabase(tx, qdbid); err != nil { + return nil, errors.Wrapf(err, "balancing database: %s", qdbid) + } else { + diffs.Merge(diff) + } + + return diffs.Output(), nil +} + +func (b *Balancer) removeDatabaseWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) (InternalDiffs, error) { + jobs, err := b.current.ListJobs(tx, roleType, qdbid, addr) + if err != nil { + return nil, errors.Wrap(err, "listing jobs") + } + + // Before removing the worker, mark its jobs as free. + if err := b.freeJobs.MergeJobs(tx, roleType, qdbid, jobs); err != nil { + return nil, errors.Wrap(err, "merging free jobs") + } + + // Remove the worker. + if err := b.current.DeleteWorker(tx, roleType, qdbid, addr); err != nil { + return nil, errors.Wrap(err, "deleting worker") + } + + // Even though this may not be useful to the caller (for example, in the + // case where the worker has died and no longer exists), return the diffs + // which represent the removal of jobs from the worker. + diff := NewInternalDiffs() + for _, job := range jobs { + diff.Removed(addr, job) + } + + return diff, nil +} + +func (b *Balancer) AddJobs(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID, jobs ...dax.Job) ([]dax.WorkerDiff, error) { + start := time.Now() + defer func() { + log.Printf("ELAPSED: Balancer.AddJob: %v", time.Since(start)) + }() + + switch len(jobs) { + case 0: + // No jobs so return early. + b.logger.Debugf("%s: AddJobs (no jobs provided)", roleType) + return []dax.WorkerDiff{}, nil + case 1: + b.logger.Debugf("%s: AddJobs (%s)", roleType, jobs[0]) + default: + b.logger.Debugf("%s: AddJobs (%d)", roleType, len(jobs)) + } + + // TODO(tlt): we don't currently use "table" in this method; even though we + // pass a table, we're still encoding the tableKey in the job. In theory, we + // could exclude tableKey from the job coming into this method, and add it + // here. + qdbid := qtid.QualifiedDatabaseID + + diff, err := b.addJobs(tx, roleType, qdbid, jobs...) + if err != nil { + return nil, errors.Wrap(err, "adding job") + } + + return diff.Output(), nil +} + +func (b *Balancer) addJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs ...dax.Job) (InternalDiffs, error) { + diffs := NewInternalDiffs() + + if len(jobs) == 0 { + return diffs, nil + } + + if cnt, err := b.current.WorkerCount(tx, roleType, qdbid); err != nil { + return nil, errors.Wrap(err, "getting worker count") + } else if cnt == 0 { + if err := b.freeJobs.CreateJobs(tx, roleType, qdbid, jobs...); err != nil { + return nil, errors.Wrap(err, "creating free job") + } + + // Since we've just added free jobs to the database, try to add a worker + // for the database. This case would happen because when a database is + // first created, it is not assigned any workers. A database is not + // assigned workers until it has at least one job (which this database + // now has). + if diff, err := b.balanceDatabaseForRole(tx, roleType, qdbid); err != nil { + return nil, errors.Wrapf(err, "assigning min workers: (%s)", roleType) + } else { + diffs.Merge(diff) + } + + // Now check, again, to see if the database has a worker. + if cnt2, err := b.current.WorkerCount(tx, roleType, qdbid); err != nil { + return nil, errors.Wrap(err, "getting worker count, again") + } else if cnt2 == 0 { + // TODO: we might want to inform the user that a job is in the free list + // because there are no workers. + return InternalDiffs{}, nil + } + } + + diff, err := b.addDatabaseJobs(tx, roleType, qdbid, jobs...) + if err != nil { + return nil, errors.Wrapf(err, "adding database jobs: (%s) %s", roleType, qdbid) + } + diffs.Merge(diff) + + return diffs, nil +} + +// addDatabaseJobs adds the job for the provided database. +func (b *Balancer) addDatabaseJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs ...dax.Job) (InternalDiffs, error) { + workerJobs, err := b.current.WorkersJobs(tx, roleType, qdbid) + if err != nil { + return nil, errors.Wrapf(err, "getting workers jobs: %s", roleType) + } + jset := dax.NewSet[dax.Job]() + for _, workerInfo := range workerJobs { + jset.Merge(dax.NewSet(workerInfo.Jobs...)) + } + + addrs := make(dax.Addresses, 0, len(workerJobs)) + jobCounts := make(map[dax.Address]int, 0) + for _, v := range workerJobs { + addrs = append(addrs, v.Address) + jobCounts[v.Address] = len(v.Jobs) + } + + diffs := NewInternalDiffs() + + jobsToCreate := make(map[dax.Address][]dax.Job) + + for _, job := range jobs { + // Skip any job that already exists. + if jset.Contains(job) { + continue + } + + // Find the worker with the fewest number of jobs and assign it this job. + var lowCount int = math.MaxInt + var lowWorker dax.Address + + // We loop over addrs here instead of jobCounts because jobCounts is a + // map and it can return results in an unexpected order, which is a + // problem for testing. + for _, addr := range addrs { + jobCount := jobCounts[addr] + if jobCount < lowCount { + lowCount = jobCount + lowWorker = addr + } + } + + jobsToCreate[lowWorker] = append(jobsToCreate[lowWorker], job) + jobCounts[lowWorker]++ + } + + for worker, jobs := range jobsToCreate { + if err := b.current.CreateJobs(tx, roleType, qdbid, worker, jobs...); err != nil { + return nil, errors.Wrap(err, "creating job") + } + for _, job := range jobs { + diffs.Added(worker, job) + } + } + + return diffs, nil +} + +func (b *Balancer) RemoveJobs(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID, jobs ...dax.Job) ([]dax.WorkerDiff, error) { + qdbid := qtid.QualifiedDatabaseID + + // If no jobs are provided, remove all jobs for table. + if len(jobs) == 0 { + diffs, err := b.removeJobsForTable(tx, roleType, qtid) + if err != nil { + return nil, errors.Wrapf(err, "removing jobs for table: (%s) %s", roleType, qtid) + } + return diffs.Output(), nil + } + + diffs := NewInternalDiffs() + + for _, job := range jobs { + if diff, err := b.removeJob(tx, roleType, qdbid, job); err != nil { + return nil, errors.Wrapf(err, "removing job: (%s) %s, %s", roleType, qdbid, job) + } else { + diffs.Merge(diff) + } + } + + return diffs.Output(), nil +} + +func (b *Balancer) removeJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) (InternalDiffs, error) { + idiffs, err := b.current.DeleteJobsForTable(tx, roleType, qtid) + if err != nil { + return nil, errors.Wrapf(err, "deleting jobs for table: (%s) %s", roleType, qtid) + } + if err := b.freeJobs.DeleteJobsForTable(tx, roleType, qtid); err != nil { + return nil, errors.Wrapf(err, "deleting free jobs for table: (%s) %s", roleType, qtid) + } + return idiffs, nil +} + +// Balance calls balanceDatabase on every database in the schemar. +func (b *Balancer) Balance(tx dax.Transaction) ([]dax.WorkerDiff, error) { + diffs, err := b.balance(tx) + if err != nil { + return nil, errors.Wrapf(err, "balancing all") + } + + return diffs.Output(), nil +} + +func (b *Balancer) balance(tx dax.Transaction) (InternalDiffs, error) { + qdbs, err := b.schemar.Databases(tx, "") + if err != nil { + return nil, errors.Wrapf(err, "getting all databases") + } + + diffs := NewInternalDiffs() + + for _, qdb := range qdbs { + qdbid := qdb.QualifiedID() + if diff, err := b.balanceDatabase(tx, qdbid); err != nil { + return nil, errors.Wrapf(err, "balancing database: %s", qdbid) + } else { + diffs.Merge(diff) + } + } + + return diffs, nil +} + +func (b *Balancer) BalanceDatabase(tx dax.Transaction, qdbid dax.QualifiedDatabaseID) ([]dax.WorkerDiff, error) { + diffs, err := b.balanceDatabase(tx, qdbid) + if err != nil { + return nil, errors.Wrapf(err, "balancing database: %s", qdbid) + } + + return diffs.Output(), nil +} + +func (b *Balancer) balanceDatabase(tx dax.Transaction, qdbid dax.QualifiedDatabaseID) (InternalDiffs, error) { + diffs := NewInternalDiffs() + + for _, role := range []dax.RoleType{dax.RoleTypeCompute, dax.RoleTypeTranslate} { + diff, err := b.balanceDatabaseForRole(tx, role, qdbid) + if err != nil { + return nil, errors.Wrapf(err, "getting worker count: (%s) %s", role, qdbid) + } + diffs.Merge(diff) + } + + return diffs, nil +} + +func (b *Balancer) balanceDatabaseForRole(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (InternalDiffs, error) { + diffs := NewInternalDiffs() + + // Before balancing, make sure the database has its minimum number of + // workers satisfied. + // TODO(tlt): make assignMinWorkers database specific. + if diff, err := b.assignMinWorkers(tx, roleType); err != nil { + return nil, errors.Wrapf(err, "assigning min workers: (%s) %s", roleType, qdbid) + } else { + diffs.Merge(diff) + } + + // If there are no workers, we can't properly balance. + if cnt, err := b.current.WorkerCount(tx, roleType, qdbid); err != nil { + return nil, errors.Wrapf(err, "getting worker count: (%s) %s", roleType, qdbid) + } else if cnt == 0 { + return InternalDiffs{}, nil + } + + // Process the freeJobs. + if diff, err := b.processFreeJobs(tx, roleType, qdbid); err != nil { + return nil, errors.Wrapf(err, "processing free jobs: (%s) %s", roleType, qdbid) + } else { + diffs.Merge(diff) + } + + // Balance the jobs among workers. + diff, err := b.balanceDatabaseJobs(tx, roleType, qdbid, diffs) + if err != nil { + return nil, errors.Wrap(err, "balancing jobs") + } + + return diff, nil +} + +func (b *Balancer) CurrentState(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) ([]dax.WorkerInfo, error) { + return b.current.WorkersJobs(tx, roleType, qdbid) +} + +func (b *Balancer) WorkerState(tx dax.Transaction, roleType dax.RoleType, addr dax.Address) (dax.WorkerInfo, error) { + info := dax.WorkerInfo{ + Address: addr, + } + + dbkey := b.current.DatabaseForWorker(tx, addr) + if dbkey == "" { + return info, nil + } + qdbid := dbkey.QualifiedDatabaseID() + + jobs, err := b.current.ListJobs(tx, roleType, qdbid, addr) + if err != nil { + return dax.WorkerInfo{}, errors.Wrapf(err, "listing jobs: (%s) %s, %s", roleType, qdbid, addr) + } + info.Jobs = jobs + + return info, nil +} + +func (b *Balancer) WorkersForJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs ...dax.Job) ([]dax.WorkerInfo, error) { + out := make(map[dax.Address]dax.Set[dax.Job]) + + workerJobs, err := b.current.WorkersJobs(tx, roleType, qdbid) + if err != nil { + return nil, errors.Wrapf(err, "getting worker jobs: (%s) %s", roleType, qdbid) + } + for _, workerInfo := range workerJobs { + jset := dax.NewSet(workerInfo.Jobs...) + + matches := dax.NewSet[dax.Job]() + for _, job := range jobs { + if jset.Contains(job) { + matches.Add(job) + } + } + + if len(matches) > 0 { + out[workerInfo.Address] = matches + } + } + + workers := make([]dax.WorkerInfo, 0, len(out)) + + for addr, jset := range out { + workers = append(workers, dax.WorkerInfo{ + Address: addr, + Jobs: jset.Sorted(), + }) + } + + sort.Sort(dax.WorkerInfos(workers)) + + return workers, nil +} + +func (b *Balancer) WorkersForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) ([]dax.WorkerInfo, error) { + out := make(map[dax.Address]dax.Set[dax.Job]) + + qdbid := qtid.QualifiedDatabaseID + + prefix := string(qtid.Key()) + + workerJobs, err := b.current.WorkersJobs(tx, roleType, qdbid) + if err != nil { + return nil, errors.Wrapf(err, "getting worker jobs: (%s) %s", roleType, qdbid) + } + for _, workerInfo := range workerJobs { + + matches := dax.NewSet[dax.Job]() + for _, job := range workerInfo.Jobs { + if strings.HasPrefix(string(job), prefix) { + matches.Add(job) + } + } + if len(matches) > 0 { + out[workerInfo.Address] = matches + } + } + + workers := make([]dax.WorkerInfo, 0, len(out)) + + for addr, jset := range out { + workers = append(workers, dax.WorkerInfo{ + Address: addr, + Jobs: jset.Sorted(), + }) + } + + sort.Sort(dax.WorkerInfos(workers)) + + return workers, nil +} + +func (b *Balancer) removeJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, job dax.Job) (InternalDiffs, error) { + if addr, ok, err := b.workerForJob(tx, roleType, qdbid, job); err != nil { + return nil, errors.Wrapf(err, "getting worker for job: %s", job) + } else if ok { + if err := b.current.DeleteJob(tx, roleType, qdbid, addr, job); err != nil { + return nil, errors.Wrapf(err, "deleting job: (%s) %s, %s, %s", roleType, qdbid, addr, job) + } + + diffs := NewInternalDiffs() + diffs.Removed(addr, job) + + return diffs, nil + } + + // Just in case the job is in the free list (and wasn't assigned to a + // worker), remove it; there's no need to provide a diff. There should never + // be a case where the same job is both in the free list and assigned to a + // worker. + if err := b.freeJobs.DeleteJob(tx, roleType, qdbid, job); err != nil { + return nil, errors.Wrapf(err, "deleting free job: (%s) %s, %s", roleType, qdbid, job) + } + + return InternalDiffs{}, nil +} + +// balanceDatabaseJobs moves jobs among workers with the goal of having an equal +// number of jobs per worker. This method takes an `internalDiffs` as input for +// cases where some action has preceeded this call which also resulted in +// `internalDiffs`. Instead of having this method take a value, we could rely on +// the internalDiffs.merge() method, but we would need to modify that method to +// be smarter about the order in which it applies the add/remove operations. +// Until that's in place, we'll pass in a value here. +func (b *Balancer) balanceDatabaseJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, diffs InternalDiffs) (InternalDiffs, error) { + numWorkers, err := b.current.WorkerCount(tx, roleType, qdbid) + if err != nil { + return nil, errors.Wrapf(err, "getting worker count: (%s) %s", roleType, qdbid) + } + numJobs := 0 + if addrs, err := b.current.ListWorkers(tx, roleType, qdbid); err != nil { + return nil, errors.Wrapf(err, "listing workers: (%s) %s", roleType, qdbid) + } else { + for _, addr := range addrs { + jobCounts, err := b.current.JobCounts(tx, roleType, qdbid, addr) + if err != nil { + return nil, errors.Wrapf(err, "getting job count: (%s) %s, %s", roleType, qdbid, addr) + } + numJobs += jobCounts[addr] + } + } + + minJobsPerWorker := numJobs / numWorkers + numWorkersAboveMin := numJobs % numWorkers + + // workerInfos is used now in order to guarantee a sort order. + workerInfos, err := b.CurrentState(tx, roleType, qdbid) + if err != nil { + return nil, errors.Wrapf(err, "getting current state: (%s) %s", roleType, qdbid) + } + + // Loop through each worker, and if the number of jobs for the worker + // exceeds the target, then remove the job and add it back (which is + // effectively how we rebalance a job). + for i, workerInfo := range workerInfos { + numTargetJobs := minJobsPerWorker + if i < numWorkersAboveMin { + numTargetJobs += 1 + } + + jobCounts, err := b.current.JobCounts(tx, roleType, qdbid, workerInfo.Address) + if err != nil { + return nil, errors.Wrapf(err, "getting job count: (%s) %s, %s", roleType, qdbid, workerInfo.Address) + } + numCurrentJobs := jobCounts[workerInfo.Address] + + // If we don't need to remove jobs from this worker, then just continue + // on to the next worker. + if numCurrentJobs <= numTargetJobs { + continue + } + + sortedJobs, err := b.current.ListJobs(tx, roleType, qdbid, workerInfo.Address) + if err != nil { + return nil, errors.Wrapf(err, "listing jobs: (%s) %s, %s", roleType, qdbid, workerInfo.Address) + } + + // Remove the extra jobs from the end of the list, and add them back + // again (which should place them on a worker with fewer jobs). + for i := numCurrentJobs - 1; i >= numTargetJobs; i-- { + if rj, err := b.removeJob(tx, roleType, qdbid, sortedJobs[i]); err != nil { + return nil, errors.Wrapf(err, "removing job: %s", sortedJobs[i]) + } else { + diffs.Merge(rj) + } + if aj, err := b.addJobs(tx, roleType, qdbid, sortedJobs[i]); err != nil { + return nil, errors.Wrapf(err, "adding job: %s", sortedJobs[i]) + } else { + diffs.Merge(aj) + } + } + } + + return diffs, nil +} + +// processFreeJobs assigns all jobs in the free list to a worker. +func (b *Balancer) processFreeJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (InternalDiffs, error) { + diffs := NewInternalDiffs() + jobs, err := b.freeJobs.ListJobs(tx, roleType, qdbid) + if err != nil { + return nil, errors.Wrapf(err, "listing free jobs: %s", roleType) + } + for _, job := range jobs { + if aj, err := b.addDatabaseJobs(tx, roleType, qdbid, job); err != nil { + return nil, errors.Wrapf(err, "adding job: %s", job) + } else { + diffs.Merge(aj) + } + if err := b.freeJobs.DeleteJob(tx, roleType, qdbid, job); err != nil { + return nil, errors.Wrapf(err, "deleting free job: %s", job) + } + } + return diffs, nil +} + +// workerForJob returns the worker currently assigned to the given job. +func (b *Balancer) workerForJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, job dax.Job) (dax.Address, bool, error) { + workerJobs, err := b.current.WorkersJobs(tx, roleType, qdbid) + if err != nil { + return "", false, errors.Wrapf(err, "getting workers jobs: (%s) %s", roleType, qdbid) + } + for _, workerInfo := range workerJobs { + jset := dax.NewSet(workerInfo.Jobs...) + if jset.Contains(job) { + return workerInfo.Address, true, nil + } + } + return "", false, nil +} + +type WorkerJobService interface { + WorkersJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) ([]dax.WorkerInfo, error) + + WorkerCount(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (int, error) + ListWorkers(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Addresses, error) + + CreateWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error + DeleteWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error + + CreateJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, job ...dax.Job) error + DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, job dax.Job) error + DeleteJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) (InternalDiffs, error) + JobCounts(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr ...dax.Address) (map[dax.Address]int, error) + ListJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) (dax.Jobs, error) + + DatabaseForWorker(tx dax.Transaction, addr dax.Address) dax.DatabaseKey +} + +type FreeJobService interface { + CreateJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, job ...dax.Job) error + DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, job dax.Job) error + DeleteJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) error + ListJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Jobs, error) + MergeJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs dax.Jobs) error +} + +type FreeWorkerService interface { + AddWorkers(tx dax.Transaction, roleType dax.RoleType, addrs ...dax.Address) error + RemoveWorker(tx dax.Transaction, roleType dax.RoleType, addr dax.Address) error + PopWorkers(tx dax.Transaction, roleType dax.RoleType, num int) ([]dax.Address, error) + ListWorkers(tx dax.Transaction, roleType dax.RoleType) (dax.Addresses, error) +} diff --git a/dax/mds/controller/balancer/balancer_test.go b/dax/mds/controller/balancer/balancer_test.go new file mode 100644 index 000000000..e538ef388 --- /dev/null +++ b/dax/mds/controller/balancer/balancer_test.go @@ -0,0 +1,1104 @@ +package balancer_test + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/featurebasedb/featurebase/v3/dax" + daxbolt "github.com/featurebasedb/featurebase/v3/dax/boltdb" + "github.com/featurebasedb/featurebase/v3/dax/mds/controller" + "github.com/featurebasedb/featurebase/v3/dax/mds/controller/balancer/boltdb" + schemardb "github.com/featurebasedb/featurebase/v3/dax/mds/schemar/boltdb" + daxtest "github.com/featurebasedb/featurebase/v3/dax/test" + testbolt "github.com/featurebasedb/featurebase/v3/dax/test/boltdb" + "github.com/featurebasedb/featurebase/v3/logger" + "github.com/stretchr/testify/assert" +) + +func newBoltBalancer(t *testing.T) (*daxbolt.DB, func()) { + db := testbolt.MustOpenDB(t) + assert.NoError(t, db.InitializeBuckets(boltdb.BalancerBuckets...)) + assert.NoError(t, db.InitializeBuckets(schemardb.SchemarBuckets...)) + + return db, func() { + testbolt.MustCloseDB(t, db) + testbolt.CleanupDB(t, db.Path()) + } +} + +type runner interface { + run(tx dax.Transaction, bal controller.Balancer) ([]dax.WorkerDiff, error) +} + +type addWorker struct { + node *dax.Node +} + +func (r *addWorker) run(tx dax.Transaction, bal controller.Balancer) ([]dax.WorkerDiff, error) { + return bal.AddWorker(tx, r.node) +} + +type removeWorker struct { + addr dax.Address +} + +func (r *removeWorker) run(tx dax.Transaction, bal controller.Balancer) ([]dax.WorkerDiff, error) { + return bal.RemoveWorker(tx, r.addr) +} + +type addJob struct { + roleType dax.RoleType + qtid dax.QualifiedTableID + job dax.Job +} + +func (r *addJob) run(tx dax.Transaction, bal controller.Balancer) ([]dax.WorkerDiff, error) { + return bal.AddJobs(tx, r.roleType, r.qtid, r.job) +} + +type removeJob struct { + roleType dax.RoleType + qtid dax.QualifiedTableID + job dax.Job +} + +func (r *removeJob) run(tx dax.Transaction, bal controller.Balancer) ([]dax.WorkerDiff, error) { + return bal.RemoveJobs(tx, r.roleType, r.qtid, r.job) +} + +type balanceDatabase struct { + qdbid dax.QualifiedDatabaseID +} + +func (r *balanceDatabase) run(tx dax.Transaction, bal controller.Balancer) ([]dax.WorkerDiff, error) { + return bal.BalanceDatabase(tx, r.qdbid) +} + +func TestBalancer(t *testing.T) { + ctx := context.Background() + orgID := dax.OrganizationID("acme") + dbID := dax.DatabaseID("db1") + dbName := dax.DatabaseName("db1name") + tableID := dax.TableID("tbl1") + qtid := dax.NewQualifiedTableID(dax.NewQualifiedDatabaseID(orgID, dbID), tableID) + + t.Run("SingleWorker", func(t *testing.T) { + qdb := &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: dax.Database{ + ID: dbID, + Name: dbName, + Options: dax.DatabaseOptions{ + WorkersMin: 1, + WorkersMax: 1, + }, + }, + } + + schemar, cleanup := daxtest.NewSchemar(t) + defer cleanup() + + db, cleanup := newBoltBalancer(t) + defer cleanup() + bal := boltdb.NewBalancer(db, schemar, logger.NewStandardLogger(os.Stderr)) + + tx, err := db.BeginTx(ctx, true) + assert.NoError(t, err) + defer tx.Rollback() + + assert.NoError(t, schemar.CreateDatabase(tx, qdb)) + + role := dax.RoleTypeCompute + + tests := []struct { + runner runner + expDiff []dax.WorkerDiff + expState []dax.WorkerInfo + }{ + { + // Add job. + runner: &addJob{ + roleType: role, + qtid: qtid, + job: "s2", + }, + expDiff: []dax.WorkerDiff{}, + expState: []dax.WorkerInfo{}, + }, + { + // Add worker. + runner: &addWorker{ + node: &dax.Node{ + Address: "w1", + RoleTypes: dax.RoleTypes{role}, + }, + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w1", + AddedJobs: []dax.Job{"s2"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w1", + Jobs: []dax.Job{"s2"}, + }, + }, + }, + { + // Add another job out of order. + runner: &addJob{ + roleType: role, + qtid: qtid, + job: "s1", + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w1", + AddedJobs: []dax.Job{"s1"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w1", + Jobs: []dax.Job{"s1", "s2"}, + }, + }, + }, + { + // Add another job. + runner: &addJob{ + roleType: role, + qtid: qtid, + job: "s3", + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w1", + AddedJobs: []dax.Job{"s3"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w1", + Jobs: []dax.Job{"s1", "s2", "s3"}, + }, + }, + }, + { + // Add a duplicate job. + runner: &addJob{ + roleType: role, + qtid: qtid, + job: "s2", + }, + expDiff: []dax.WorkerDiff{}, + expState: []dax.WorkerInfo{ + { + Address: "w1", + Jobs: []dax.Job{"s1", "s2", "s3"}, + }, + }, + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + diff, err := test.runner.run(tx, bal) + assert.NoError(t, err) + assert.Equal(t, test.expDiff, diff) + + state, err := bal.CurrentState(tx, role, qdb.QualifiedID()) + assert.NoError(t, err) + assert.Equal(t, test.expState, state) + }) + } + + assert.NoError(t, tx.Commit()) + }) + + t.Run("MultipleWorkers", func(t *testing.T) { + dbOptions := dax.DatabaseOptions{ + WorkersMin: 2, + WorkersMax: 2, + } + + qdb := &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: dax.Database{ + ID: dbID, + Name: dbName, + Options: dbOptions, + }, + } + + schemar, cleanup := daxtest.NewSchemar(t) + defer cleanup() + + db, cleanup := newBoltBalancer(t) + defer cleanup() + bal := boltdb.NewBalancer(db, schemar, logger.NewStandardLogger(os.Stderr)) + + tx, err := db.BeginTx(ctx, true) + assert.NoError(t, err) + defer tx.Rollback() + + assert.NoError(t, schemar.CreateDatabase(tx, qdb)) + + role := dax.RoleTypeCompute + + type testPart struct { + name string + runner runner + expDiff []dax.WorkerDiff + expState []dax.WorkerInfo + } + + runTestPart := func(tp testPart) { + t.Run(fmt.Sprintf("test-%s", tp.name), func(t *testing.T) { + diff, err := tp.runner.run(tx, bal) + assert.NoError(t, err) + assert.Equal(t, tp.expDiff, diff) + + state, err := bal.CurrentState(tx, role, qdb.QualifiedID()) + assert.NoError(t, err) + assert.Equal(t, tp.expState, state) + }) + } + + runTestPart(testPart{ + name: "balance when empty", + runner: &balanceDatabase{ + qdbid: qdb.QualifiedID(), + }, + expDiff: []dax.WorkerDiff{}, + expState: []dax.WorkerInfo{}, + }) + + runTestPart(testPart{ + name: "add worker", + runner: &addWorker{ + node: &dax.Node{ + Address: "w2", + RoleTypes: dax.RoleTypes{role}, + }, + }, + expDiff: []dax.WorkerDiff{}, + expState: []dax.WorkerInfo{}, + }) + + runTestPart(testPart{ + name: "add worker again", + runner: &addWorker{ + node: &dax.Node{ + Address: "w2", + RoleTypes: dax.RoleTypes{role}, + }, + }, + expDiff: []dax.WorkerDiff{}, + expState: []dax.WorkerInfo{}, + }) + + runTestPart(testPart{ + name: "add a second worker", + runner: &addWorker{ + node: &dax.Node{ + Address: "w1", + RoleTypes: dax.RoleTypes{role}, + }, + }, + expDiff: []dax.WorkerDiff{}, + expState: []dax.WorkerInfo{}, + }) + + runTestPart(testPart{ + name: "add job 2", + runner: &addJob{ + roleType: role, + qtid: qtid, + job: "s2", + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w1", + AddedJobs: []dax.Job{"s2"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w1", + Jobs: []dax.Job{"s2"}, + }, + { + Address: "w2", + Jobs: []dax.Job{}, + }, + }, + }) + + runTestPart(testPart{ + name: "add job 3", + runner: &addJob{ + roleType: role, + qtid: qtid, + job: "s3", + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w2", + AddedJobs: []dax.Job{"s3"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w1", + Jobs: []dax.Job{"s2"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s3"}, + }, + }, + }) + + runTestPart(testPart{ + name: "add job 1", + runner: &addJob{ + roleType: role, + qtid: qtid, + job: "s1", + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w1", + AddedJobs: []dax.Job{"s1"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w1", + Jobs: []dax.Job{"s1", "s2"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s3"}, + }, + }, + }) + + // Update database options on schemar so min worker for db is 3. Here we + // are updating the database option first, and then adding a worker to + // satisfy those options. + t.Run(fmt.Sprintf("test-%s", "set database options db min 3"), func(t *testing.T) { + dbOptions.WorkersMin = 3 + dbOptions.WorkersMax = 3 + assert.NoError(t, schemar.SetDatabaseOptions(tx, qdb.QualifiedID(), dbOptions)) + }) + + runTestPart(testPart{ + name: "add a third worker", + runner: &addWorker{ + node: &dax.Node{ + Address: "w0", + RoleTypes: dax.RoleTypes{role}, + }, + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w0", + AddedJobs: []dax.Job{"s2"}, + RemovedJobs: []dax.Job{}, + }, + { + Address: "w1", + AddedJobs: []dax.Job{}, + RemovedJobs: []dax.Job{"s2"}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w0", + Jobs: []dax.Job{"s2"}, + }, + { + Address: "w1", + Jobs: []dax.Job{"s1"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s3"}, + }, + }, + }) + + runTestPart(testPart{ + name: "add job 4", + runner: &addJob{ + roleType: role, + qtid: qtid, + job: "s4", + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w0", + AddedJobs: []dax.Job{"s4"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w0", + Jobs: []dax.Job{"s2", "s4"}, + }, + { + Address: "w1", + Jobs: []dax.Job{"s1"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s3"}, + }, + }, + }) + + runTestPart(testPart{ + name: "add job 5", + runner: &addJob{ + roleType: role, + qtid: qtid, + job: "s5", + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w1", + AddedJobs: []dax.Job{"s5"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w0", + Jobs: []dax.Job{"s2", "s4"}, + }, + { + Address: "w1", + Jobs: []dax.Job{"s1", "s5"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s3"}, + }, + }, + }) + + runTestPart(testPart{ + name: "add job 0", + runner: &addJob{ + roleType: role, + qtid: qtid, + job: "s0", + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w2", + AddedJobs: []dax.Job{"s0"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w0", + Jobs: []dax.Job{"s2", "s4"}, + }, + { + Address: "w1", + Jobs: []dax.Job{"s1", "s5"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s0", "s3"}, + }, + }, + }) + + runTestPart(testPart{ + name: "add job 6", + runner: &addJob{ + roleType: role, + qtid: qtid, + job: "s6", + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w0", + AddedJobs: []dax.Job{"s6"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w0", + Jobs: []dax.Job{"s2", "s4", "s6"}, + }, + { + Address: "w1", + Jobs: []dax.Job{"s1", "s5"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s0", "s3"}, + }, + }, + }) + + runTestPart(testPart{ + name: "add job 7", + runner: &addJob{ + roleType: role, + qtid: qtid, + job: "s7", + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w1", + AddedJobs: []dax.Job{"s7"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w0", + Jobs: []dax.Job{"s2", "s4", "s6"}, + }, + { + Address: "w1", + Jobs: []dax.Job{"s1", "s5", "s7"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s0", "s3"}, + }, + }, + }) + + //////////////////// Remove ///////////////////////// + + runTestPart(testPart{ + name: "remove nonexistent worker", + runner: &removeWorker{ + addr: "nonexistent", + }, + expDiff: []dax.WorkerDiff{}, + expState: []dax.WorkerInfo{ + { + Address: "w0", + Jobs: []dax.Job{"s2", "s4", "s6"}, + }, + { + Address: "w1", + Jobs: []dax.Job{"s1", "s5", "s7"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s0", "s3"}, + }, + }, + }) + + runTestPart(testPart{ + name: "remove worker", + runner: &removeWorker{ + addr: "w1", + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w0", + AddedJobs: []dax.Job{"s5"}, + RemovedJobs: []dax.Job{}, + }, + { + Address: "w1", + AddedJobs: []dax.Job{}, + RemovedJobs: []dax.Job{"s1", "s5", "s7"}, + }, + { + Address: "w2", + AddedJobs: []dax.Job{"s1", "s7"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w0", + Jobs: []dax.Job{"s2", "s4", "s5", "s6"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s0", "s1", "s3", "s7"}, + }, + }, + }) + + runTestPart(testPart{ + name: "remove active job", + runner: &removeJob{ + roleType: role, + qtid: qtid, + job: "s0", + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w2", + AddedJobs: []dax.Job{}, + RemovedJobs: []dax.Job{"s0"}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w0", + Jobs: []dax.Job{"s2", "s4", "s5", "s6"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s1", "s3", "s7"}, + }, + }, + }) + + // Set WorkersMin back to 2 so we can test the change to 3 again. + t.Run(fmt.Sprintf("test-%s", "set database options db min back to 2"), func(t *testing.T) { + dbOptions.WorkersMin = 2 + dbOptions.WorkersMax = 2 + assert.NoError(t, schemar.SetDatabaseOptions(tx, qdb.QualifiedID(), dbOptions)) + }) + + runTestPart(testPart{ + name: "add a fourth worker", + runner: &addWorker{ + node: &dax.Node{ + Address: "w3", + RoleTypes: dax.RoleTypes{role}, + }, + }, + expDiff: []dax.WorkerDiff{}, + expState: []dax.WorkerInfo{ + { + Address: "w0", + Jobs: []dax.Job{"s2", "s4", "s5", "s6"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s1", "s3", "s7"}, + }, + }, + }) + + // Update database options on schemar so min worker for db is 4. Here we + // have added a worker which will satisfy this option, and then updated + // the option. + t.Run(fmt.Sprintf("test-%s", "set database options db min back to 3"), func(t *testing.T) { + dbOptions.WorkersMin = 3 + dbOptions.WorkersMax = 3 + assert.NoError(t, schemar.SetDatabaseOptions(tx, qdb.QualifiedID(), dbOptions)) + }) + + // This implies that there is a condition where the database does not + // get repaired to its workerMin: if workers per database has dropped + // below its min, and there are no available workers to replace the + // missing workers (and bring it back to min), then the database will + // operate below min. If, then, a worker is added to the pool, we do not + // currently have a process to automatically assign that worker to a + // database under min. That happens with an explicit call to + // assignMinWorkers or BalanceDatabase. + runTestPart(testPart{ + name: "balance to include new third worker", + runner: &balanceDatabase{ + qdbid: qdb.QualifiedID(), + }, + expDiff: []dax.WorkerDiff{ + { + Address: "w0", + AddedJobs: []dax.Job{}, + RemovedJobs: []dax.Job{"s6"}, + }, + { + Address: "w2", + AddedJobs: []dax.Job{}, + RemovedJobs: []dax.Job{"s7"}, + }, + { + Address: "w3", + AddedJobs: []dax.Job{"s6", "s7"}, + RemovedJobs: []dax.Job{}, + }, + }, + expState: []dax.WorkerInfo{ + { + Address: "w0", + Jobs: []dax.Job{"s2", "s4", "s5"}, + }, + { + Address: "w2", + Jobs: []dax.Job{"s1", "s3"}, + }, + { + Address: "w3", + Jobs: []dax.Job{"s6", "s7"}, + }, + }, + }) + + assert.NoError(t, tx.Commit()) + }) + + t.Run("WorkersForJobs", func(t *testing.T) { + qdb := &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: dax.Database{ + ID: dbID, + Name: dbName, + Options: dax.DatabaseOptions{ + WorkersMin: 2, + WorkersMax: 2, + }, + }, + } + qdbid := qdb.QualifiedID() + + schemar, scleanup := daxtest.NewSchemar(t) + defer scleanup() + + role := dax.RoleTypeCompute + + db, cleanup := newBoltBalancer(t) + defer cleanup() + bal := boltdb.NewBalancer(db, schemar, logger.NewStandardLogger(os.Stderr)) + + tx, err := db.BeginTx(ctx, true) + assert.NoError(t, err) + defer tx.Rollback() + + assert.NoError(t, schemar.CreateDatabase(tx, qdb)) + + node1 := &dax.Node{ + Address: "n1", + RoleTypes: []dax.RoleType{role}, + } + + node2 := &dax.Node{ + Address: "n2", + RoleTypes: []dax.RoleType{role}, + } + + _, err = bal.AddWorker(tx, node1) + assert.NoError(t, err) + _, err = bal.AddWorker(tx, node2) + assert.NoError(t, err) + for i := 0; i < 12; i++ { + _, err = bal.AddJobs(tx, role, qtid, dax.Job(fmt.Sprintf("s%d", i))) + assert.NoError(t, err) + } + + exp := dax.WorkerInfo{ + Address: "n1", + Jobs: []dax.Job{"s0", "s10", "s2", "s4", "s6", "s8"}, + } + ws, err := bal.WorkerState(tx, role, "n1") + assert.NoError(t, err) + assert.Equal(t, exp, ws) + + exp = dax.WorkerInfo{ + Address: "n2", + Jobs: []dax.Job{"s1", "s11", "s3", "s5", "s7", "s9"}, + } + ws, err = bal.WorkerState(tx, role, "n2") + assert.NoError(t, err) + assert.Equal(t, exp, ws) + + tests := []struct { + jobs []dax.Job + exp []dax.WorkerInfo + }{ + { + jobs: []dax.Job{"s0"}, + exp: []dax.WorkerInfo{ + {Address: "n1", Jobs: []dax.Job{"s0"}}, + }, + }, + { + jobs: []dax.Job{"s0", "s4"}, + exp: []dax.WorkerInfo{ + {Address: "n1", Jobs: []dax.Job{"s0", "s4"}}, + }, + }, + { + jobs: []dax.Job{"s0", "s4", "s999"}, + exp: []dax.WorkerInfo{ + {Address: "n1", Jobs: []dax.Job{"s0", "s4"}}, + }, + }, + { + jobs: []dax.Job{"s0", "s1"}, + exp: []dax.WorkerInfo{ + {Address: "n1", Jobs: []dax.Job{"s0"}}, + {Address: "n2", Jobs: []dax.Job{"s1"}}, + }, + }, + { + jobs: []dax.Job{"s5", "s0", "s1", "s8"}, + exp: []dax.WorkerInfo{ + {Address: "n1", Jobs: []dax.Job{"s0", "s8"}}, + {Address: "n2", Jobs: []dax.Job{"s1", "s5"}}, + }, + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + workers, err := bal.WorkersForJobs(tx, role, qdbid, test.jobs...) + assert.NoError(t, err) + assert.Equal(t, test.exp, workers) + }) + } + + assert.NoError(t, tx.Commit()) + }) + + t.Run("WorkersForTable", func(t *testing.T) { + qdb := &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: dax.Database{ + ID: dbID, + Name: dbName, + Options: dax.DatabaseOptions{ + WorkersMin: 2, + WorkersMax: 2, + }, + }, + } + + qdbid := dax.NewQualifiedDatabaseID(orgID, dbID) + + // Table 1. + tbl1 := &dax.Table{ + ID: "id1", + Name: "table1", + } + qtbl1 := dax.NewQualifiedTable(qdbid, tbl1) + qtid1 := qtbl1.QualifiedID() + + // Table 2. + tbl2 := &dax.Table{ + ID: "id2", + Name: "table2", + } + qtbl2 := dax.NewQualifiedTable(qdbid, tbl2) + qtid2 := qtbl2.QualifiedID() + + schemar, scleanup := daxtest.NewSchemar(t) + defer scleanup() + + role := dax.RoleTypeCompute + + db, cleanup := newBoltBalancer(t) + defer cleanup() + bal := boltdb.NewBalancer(db, schemar, logger.NewStandardLogger(os.Stderr)) + + tx, err := db.BeginTx(ctx, true) + assert.NoError(t, err) + defer tx.Rollback() + + assert.NoError(t, schemar.CreateDatabase(tx, qdb)) + + node1 := &dax.Node{ + Address: "n1", + RoleTypes: []dax.RoleType{role}, + } + + node2 := &dax.Node{ + Address: "n2", + RoleTypes: []dax.RoleType{role}, + } + + _, err = bal.AddWorker(tx, node1) + assert.NoError(t, err) + _, err = bal.AddWorker(tx, node2) + assert.NoError(t, err) + for i := 0; i < 12; i++ { + job := fmt.Sprintf("%s|s%d", qtid1.Key(), i) + _, err = bal.AddJobs(tx, role, qtid1, dax.Job(job)) + assert.NoError(t, err) + } + + for i := 9; i < 16; i++ { + job := fmt.Sprintf("%s|s%d", qtid2.Key(), i) + _, err = bal.AddJobs(tx, role, qtid2, dax.Job(job)) + assert.NoError(t, err) + } + + // fn1 and fn2 are just helper functions used to make the tests easier + // to read. + fn1 := func(s string) dax.Job { + return dax.Job(fmt.Sprintf("tbl__acme__db1__id1|%s", s)) + } + fn2 := func(s string) dax.Job { + return dax.Job(fmt.Sprintf("tbl__acme__db1__id2|%s", s)) + } + + // Table 1 + workers, err := bal.WorkersForTable(tx, role, qtid1) + assert.NoError(t, err) + assert.ElementsMatch(t, []dax.WorkerInfo{ + {Address: "n1", Jobs: []dax.Job{fn1("s0"), fn1("s10"), fn1("s2"), fn1("s4"), fn1("s6"), fn1("s8")}}, + {Address: "n2", Jobs: []dax.Job{fn1("s1"), fn1("s11"), fn1("s3"), fn1("s5"), fn1("s7"), fn1("s9")}}, + }, workers) + + // Table 2 + workers, err = bal.WorkersForTable(tx, role, qtid2) + assert.NoError(t, err) + assert.ElementsMatch(t, []dax.WorkerInfo{ + {Address: "n1", Jobs: []dax.Job{fn2("s11"), fn2("s13"), fn2("s15"), fn2("s9")}}, + {Address: "n2", Jobs: []dax.Job{fn2("s10"), fn2("s12"), fn2("s14")}}, + }, workers) + + // No match + qtid0 := dax.NewQualifiedTableID(qdbid, "bad") + workers, err = bal.WorkersForTable(tx, role, qtid0) + assert.NoError(t, err) + assert.ElementsMatch(t, []dax.WorkerInfo{}, workers) + + assert.NoError(t, tx.Commit()) + }) + + t.Run("Balance", func(t *testing.T) { + dbOptions := dax.DatabaseOptions{ + WorkersMin: 2, + WorkersMax: 2, + } + qdb := &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: dax.Database{ + ID: dbID, + Name: dbName, + Options: dbOptions, + }, + } + + qdbid := qdb.QualifiedID() + + schemar, scleanup := daxtest.NewSchemar(t) + defer scleanup() + + role := dax.RoleTypeCompute + + db, cleanup := newBoltBalancer(t) + defer cleanup() + bal := boltdb.NewBalancer(db, schemar, logger.NewStandardLogger(os.Stderr)) + + tx, err := db.BeginTx(ctx, true) + assert.NoError(t, err) + defer tx.Rollback() + + assert.NoError(t, schemar.CreateDatabase(tx, qdb)) + + node1 := &dax.Node{ + Address: "n1", + RoleTypes: []dax.RoleType{role}, + } + + node2 := &dax.Node{ + Address: "n2", + RoleTypes: []dax.RoleType{role}, + } + + node3 := &dax.Node{ + Address: "n3", + RoleTypes: []dax.RoleType{role}, + } + + // Add two workers with some jobs evenly spread across them. + _, err = bal.AddWorker(tx, node1) + assert.NoError(t, err) + _, err = bal.AddWorker(tx, node2) + assert.NoError(t, err) + for i := 0; i < 13; i++ { + job := fmt.Sprintf("s%d", i) + _, err = bal.AddJobs(tx, role, qtid, dax.Job(job)) + assert.NoError(t, err) + } + + exp := dax.WorkerInfo{ + Address: "n1", + Jobs: []dax.Job{"s0", "s10", "s12", "s2", "s4", "s6", "s8"}, + } + ws, err := bal.WorkerState(tx, role, "n1") + assert.NoError(t, err) + assert.Equal(t, exp, ws) + + exp = dax.WorkerInfo{ + Address: "n2", + Jobs: []dax.Job{"s1", "s11", "s3", "s5", "s7", "s9"}, + } + ws, err = bal.WorkerState(tx, role, "n2") + assert.NoError(t, err) + assert.Equal(t, exp, ws) + + // Update database options on schemar so min worker for db is 3. + t.Run(fmt.Sprintf("test-%s", "set database options db min 3"), func(t *testing.T) { + dbOptions.WorkersMin = 3 + dbOptions.WorkersMax = 3 + assert.NoError(t, schemar.SetDatabaseOptions(tx, qdb.QualifiedID(), dbOptions)) + }) + + // Now, add a worker and confirm that it has received some jobs. + _, err = bal.AddWorker(tx, node3) + assert.NoError(t, err) + exp = dax.WorkerInfo{ + Address: "n3", + Jobs: []dax.Job{"s6", "s7", "s8", "s9"}, + } + ws, err = bal.WorkerState(tx, role, "n3") + assert.NoError(t, err) + assert.Equal(t, exp, ws) + + // Finally, call Balance() and confirm that the appropriate jobs got are + // as expected (actually, this is no longer needed here because we + // automatically balance when we add workers, but calling it should + // effectively be a no-op). + _, err = bal.BalanceDatabase(tx, qdbid) + assert.NoError(t, err) + + exp = dax.WorkerInfo{ + Address: "n1", + Jobs: []dax.Job{"s0", "s10", "s12", "s2", "s4"}, + } + ws, err = bal.WorkerState(tx, role, "n1") + assert.NoError(t, err) + assert.Equal(t, exp, ws) + + exp = dax.WorkerInfo{ + Address: "n2", + Jobs: []dax.Job{"s1", "s11", "s3", "s5"}, + } + ws, err = bal.WorkerState(tx, role, "n2") + assert.NoError(t, err) + assert.Equal(t, exp, ws) + + exp = dax.WorkerInfo{ + Address: "n3", + Jobs: []dax.Job{"s6", "s7", "s8", "s9"}, + } + ws, err = bal.WorkerState(tx, role, "n3") + assert.NoError(t, err) + assert.Equal(t, exp, ws) + + assert.NoError(t, tx.Commit()) + }) +} diff --git a/dax/mds/controller/balancer/boltdb/balancer.go b/dax/mds/controller/balancer/boltdb/balancer.go new file mode 100644 index 000000000..bb2725718 --- /dev/null +++ b/dax/mds/controller/balancer/boltdb/balancer.go @@ -0,0 +1,813 @@ +// Package boltdb contains the boltdb implementation of the Balancer interface. +package boltdb + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/dax/boltdb" + "github.com/featurebasedb/featurebase/v3/dax/mds/controller" + "github.com/featurebasedb/featurebase/v3/dax/mds/controller/balancer" + "github.com/featurebasedb/featurebase/v3/dax/mds/schemar" + "github.com/featurebasedb/featurebase/v3/errors" + "github.com/featurebasedb/featurebase/v3/logger" +) + +var ( + bucketBalancer = boltdb.Bucket("balancer") +) + +// BalancerBuckets defines the buckets used by this package. It can be +// called during setup to create the buckets ahead of time. +var BalancerBuckets []boltdb.Bucket = []boltdb.Bucket{ + bucketBalancer, +} + +// NewBalancer returns a new instance of controller.Balancer. +func NewBalancer(db *boltdb.DB, schemar schemar.Schemar, logger logger.Logger) controller.Balancer { + fjs := newFreeJobService(db) + wjs := newWorkerJobService(db, logger) + fws := newFreeWorkerService(db) + + return balancer.New(fjs, wjs, fws, schemar, logger) +} + +// Ensure type implements interface. +var _ balancer.WorkerJobService = (*workerJobService)(nil) + +type workerJobService struct { + db *boltdb.DB + logger logger.Logger +} + +func newWorkerJobService(db *boltdb.DB, logger logger.Logger) *workerJobService { + return &workerJobService{ + db: db, + logger: logger, + } +} + +func (w *workerJobService) WorkersJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) ([]dax.WorkerInfo, error) { + workerInfos, err := w.getWorkerInfos(tx, roleType, qdbid) + if err != nil { + return nil, errors.Wrapf(err, "getting worker infos: %s", roleType) + } + + return workerInfos, nil +} + +func (w *workerJobService) WorkerCount(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (int, error) { + workers, err := w.getWorkers(tx, roleType, qdbid) + if err != nil { + return 0, errors.Wrapf(err, "getting workers: %s", roleType) + } + + return len(workers), nil +} + +func (w *workerJobService) ListWorkers(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Addresses, error) { + return w.getWorkers(tx, roleType, qdbid) +} + +func (w *workerJobService) getWorkers(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Addresses, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() + } + + c := txx.Bucket(bucketBalancer).Cursor() + + // Deserialize rows into Worker objects. + addrs := make(dax.Addresses, 0) + + prefix := []byte(fmt.Sprintf(prefixFmtWorkersDB, roleType, qdbid.Key())) + for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { + if v == nil { + w.logger.Printf("nil value for key: %s", k) + continue + } + + addr, err := keyWorker(k) + if err != nil { + return nil, errors.Wrapf(err, "getting worker from key: %s", k) + } + + addrs = append(addrs, addr) + } + + return addrs, nil +} + +func (w *workerJobService) getWorkerInfos(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.WorkerInfos, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() + } + + c := txx.Bucket(bucketBalancer).Cursor() + + // Deserialize rows into WorkerInfo objects. + workerInfos := make(dax.WorkerInfos, 0) + + prefix := []byte(fmt.Sprintf(prefixFmtWorkersDB, roleType, qdbid.Key())) + for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { + addr, err := keyWorker(k) + if err != nil { + return nil, errors.Wrapf(err, "getting worker from key: %s", k) + } + + jobs := dax.NewSet[dax.Job]() + if v != nil { + jobs, err = decodeJobSet(v) + if err != nil { + return nil, errors.Wrap(err, "decoding job set") + } + } + + workerInfo := dax.WorkerInfo{ + Address: addr, + Jobs: jobs.Sorted(), + } + + workerInfos = append(workerInfos, workerInfo) + } + + return workerInfos, nil +} + +func (w *workerJobService) CreateWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + // If this worker already exists, don't do anything. + wrkr := bkt.Get(workerDBKey(roleType, qdbid, addr)) + if wrkr != nil { + return nil + } + + val := []byte("[]") + if err := bkt.Put(workerDBKey(roleType, qdbid, addr), val); err != nil { + return errors.Wrapf(err, "putting db worker: %s, %s", qdbid, addr) + } + + if err := bkt.Put(workerAssignedKey(addr), []byte(qdbid.Key())); err != nil { + return errors.Wrapf(err, "putting assigned worker: %s, %s", qdbid, addr) + } + + return nil +} + +func (w *workerJobService) DeleteWorker(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + if err := bkt.Delete(workerDBKey(roleType, qdbid, addr)); err != nil { + return errors.Wrapf(err, "deleting node key: %s", workerDBKey(roleType, qdbid, addr)) + } + + if err := bkt.Delete(workerAssignedKey(addr)); err != nil { + return errors.Wrapf(err, "deleting assigned worker: %s", workerAssignedKey(addr)) + } + + return nil +} + +func (w *workerJobService) CreateJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, jobs ...dax.Job) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + jobset := dax.NewSet[dax.Job]() + var err error + + // get worker + wrkr := bkt.Get(workerDBKey(roleType, qdbid, addr)) + if wrkr != nil { + jobset, err = decodeJobSet(wrkr) + if err != nil { + return errors.Wrap(err, "decoding job set") + } + } + + for _, job := range jobs { + jobset.Add(job) + } + val, err := encodeJobSet(jobset) + if err != nil { + return errors.Wrap(err, "encoding job set") + } + + if err := bkt.Put(workerDBKey(roleType, qdbid, addr), val); err != nil { + return errors.Wrap(err, "putting worker") + } + + return nil +} + +func (w *workerJobService) DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address, job dax.Job) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + // get worker + wrkr := bkt.Get(workerDBKey(roleType, qdbid, addr)) + if wrkr == nil { + return nil + } + + jobset, err := decodeJobSet(wrkr) + if err != nil { + return errors.Wrap(err, "decoding job set") + } + if !jobset.Contains(job) { + return nil + } + + jobset.Remove(job) + val, err := encodeJobSet(jobset) + if err != nil { + return errors.Wrap(err, "encoding job set") + } + + if err := bkt.Put(workerDBKey(roleType, qdbid, addr), val); err != nil { + return errors.Wrap(err, "putting worker") + } + + return nil +} + +func (w *workerJobService) DeleteJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) (balancer.InternalDiffs, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + qdbid := qtid.QualifiedDatabaseID + prefix := string(qtid.Key()) + + workers, err := w.getWorkers(tx, roleType, qdbid) + if err != nil { + return nil, errors.Wrap(err, "getting workers") + } + + idiffs := balancer.NewInternalDiffs() + for _, worker := range workers { + // get worker + wrkr := bkt.Get(workerDBKey(roleType, qdbid, worker)) + if wrkr == nil { + panic("didn't find worker that should... definitely exist") + } + jobset, err := decodeJobSet(wrkr) + if err != nil { + return nil, errors.Wrap(err, "decoding job set") + } + + jobs := jobset.RemoveByPrefix(prefix) + for _, job := range jobs { + idiffs.Removed(worker, job) + } + val, err := encodeJobSet(jobset) + if err != nil { + return nil, errors.Wrap(err, "encoding job set") + } + + if err := bkt.Put(workerDBKey(roleType, qdbid, worker), val); err != nil { + return nil, errors.Wrap(err, "putting worker") + } + + } + + return idiffs, nil +} + +func (w *workerJobService) ListJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) (dax.Jobs, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + jobset := dax.NewSet[dax.Job]() + var err error + + // get worker + wrkr := bkt.Get(workerDBKey(roleType, qdbid, addr)) + if wrkr != nil { + jobset, err = decodeJobSet(wrkr) + if err != nil { + return nil, errors.Wrap(err, "decoding job set") + } + } + + return jobset.Sorted(), nil +} + +func (w *workerJobService) JobCounts(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addrs ...dax.Address) (map[dax.Address]int, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + m := make(map[dax.Address]int) + + for _, addr := range addrs { + jobset := dax.NewSet[dax.Job]() + var err error + + // get worker + wrkr := bkt.Get(workerDBKey(roleType, qdbid, addr)) + if wrkr != nil { + jobset, err = decodeJobSet(wrkr) + if err != nil { + return nil, errors.Wrap(err, "decoding job set") + } + } + + m[addr] = len(jobset) + } + + return m, nil +} + +func (w *workerJobService) DatabaseForWorker(tx dax.Transaction, addr dax.Address) dax.DatabaseKey { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return "" // TODO(tlt): return error here? + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return "" + } + + wrkr := bkt.Get(workerAssignedKey(addr)) + + return dax.DatabaseKey(wrkr) +} + +// encodeJobSet encode the jobSet into a JSON array of strings. +func encodeJobSet(jobSet dax.Set[dax.Job]) ([]byte, error) { + arr := jobSet.Sorted() + b, err := json.Marshal(arr) + if err != nil { + return nil, errors.Wrap(err, "marshalling json") + } + return b, nil +} + +// decodeJobSet decode the string (a JSON array of strings) into jobSet. +func decodeJobSet(v []byte) (dax.Set[dax.Job], error) { + var arr []string + err := json.Unmarshal(v, &arr) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling json") + } + + js := dax.NewSet[dax.Job]() + for _, s := range arr { + js.Add(dax.Job(s)) + } + + return js, nil +} + +// encodeWorkerSet encode the workerSet into a JSON array of strings. +func encodeWorkerSet(workerSet dax.Set[dax.Address]) ([]byte, error) { + arr := workerSet.Sorted() + b, err := json.Marshal(arr) + if err != nil { + return nil, errors.Wrap(err, "marshalling json") + } + return b, nil +} + +// decodeWorkerSet decode the string (a JSON array of strings) into workerSet. +func decodeWorkerSet(v []byte) (dax.Set[dax.Address], error) { + var arr []string + err := json.Unmarshal(v, &arr) + if err != nil { + return nil, errors.Wrap(err, "unmarshalling json") + } + + ws := dax.NewSet[dax.Address]() + for _, s := range arr { + ws.Add(dax.Address(s)) + } + + return ws, nil +} + +// Ensure type implements interface. +var _ balancer.FreeJobService = (*freeJobService)(nil) + +type freeJobService struct { + db *boltdb.DB +} + +func newFreeJobService(db *boltdb.DB) *freeJobService { + return &freeJobService{ + db: db, + } +} + +func (f *freeJobService) CreateJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs ...dax.Job) error { + return f.MergeJobs(tx, roleType, qdbid, jobs) +} + +func (f *freeJobService) DeleteJob(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, job dax.Job) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + // get free jobs + fjs := bkt.Get(freeJobKey(roleType, qdbid)) + if fjs == nil { + return nil + } + + jobset, err := decodeJobSet(fjs) + if err != nil { + return errors.Wrap(err, "decoding job set") + } + if !jobset.Contains(job) { + return nil + } + + jobset.Remove(job) + val, err := encodeJobSet(jobset) + if err != nil { + return errors.Wrap(err, "encoding job set") + } + + if err := bkt.Put(freeJobKey(roleType, qdbid), val); err != nil { + return errors.Wrap(err, "putting free job") + } + + return nil +} + +func (f *freeJobService) DeleteJobsForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + qdbid := qtid.QualifiedDatabaseID + prefix := string(qtid.Key()) + + // get free jobs + fjs := bkt.Get(freeJobKey(roleType, qdbid)) + if fjs == nil { + return nil + } + + jobset, err := decodeJobSet(fjs) + if err != nil { + return errors.Wrap(err, "decoding job set") + } + + jobset.RemoveByPrefix(prefix) + val, err := encodeJobSet(jobset) + if err != nil { + return errors.Wrap(err, "encoding job set") + } + + if err := bkt.Put(freeJobKey(roleType, qdbid), val); err != nil { + return errors.Wrap(err, "putting free job") + } + + return nil +} + +func (f *freeJobService) ListJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) (dax.Jobs, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + jobset := dax.NewSet[dax.Job]() + var err error + + // get free jobs + fjs := bkt.Get(freeJobKey(roleType, qdbid)) + if fjs != nil { + jobset, err = decodeJobSet(fjs) + if err != nil { + return nil, errors.Wrap(err, "decoding job set") + } + } + + return jobset.Sorted(), nil +} + +func (f *freeJobService) MergeJobs(tx dax.Transaction, roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, jobs dax.Jobs) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + jobset := dax.NewSet[dax.Job]() + var err error + + // get free jobs + fjs := bkt.Get(freeJobKey(roleType, qdbid)) + if fjs != nil { + jobset, err = decodeJobSet(fjs) + if err != nil { + return errors.Wrap(err, "decoding job set") + } + } + + for _, j := range jobs { + jobset.Add(j) + } + val, err := encodeJobSet(jobset) + if err != nil { + return errors.Wrap(err, "encoding job set") + } + + if err := bkt.Put(freeJobKey(roleType, qdbid), val); err != nil { + return errors.Wrap(err, "putting free job") + } + + return nil +} + +////////////////////////////////////////////////////// + +// Ensure type implements interface. +var _ balancer.FreeWorkerService = (*freeWorkerService)(nil) + +type freeWorkerService struct { + db *boltdb.DB +} + +func newFreeWorkerService(db *boltdb.DB) *freeWorkerService { + return &freeWorkerService{ + db: db, + } +} + +func (f *freeWorkerService) AddWorkers(tx dax.Transaction, roleType dax.RoleType, addres ...dax.Address) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + workerset := dax.NewSet[dax.Address]() + var err error + + // get free workers + fws := bkt.Get(freeWorkerKey(roleType)) + if fws != nil { + workerset, err = decodeWorkerSet(fws) + if err != nil { + return errors.Wrap(err, "decoding worker set") + } + } + + for _, w := range addres { + workerset.Add(w) + } + val, err := encodeWorkerSet(workerset) + if err != nil { + return errors.Wrap(err, "encoding worker set") + } + + if err := bkt.Put(freeWorkerKey(roleType), val); err != nil { + return errors.Wrap(err, "putting free worker") + } + + return nil +} + +func (f *freeWorkerService) RemoveWorker(tx dax.Transaction, roleType dax.RoleType, addr dax.Address) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + workers, err := f.ListWorkers(tx, roleType) + if err != nil { + return errors.Wrap(err, "listing free workers") + } + + // Create a workerset containing the free workers which remain after + // removing num workers. + workerset := dax.NewSet[dax.Address]() + for _, w := range workers { + workerset.Add(w) + } + + if !workerset.Contains(addr) { + return nil + } + + // Remove the worker. + workerset.Remove(addr) + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + val, err := encodeWorkerSet(workerset) + if err != nil { + return errors.Wrap(err, "encoding worker set") + } + + if err := bkt.Put(freeWorkerKey(roleType), val); err != nil { + return errors.Wrap(err, "putting free worker") + } + + return nil +} + +func (f *freeWorkerService) PopWorkers(tx dax.Transaction, roleType dax.RoleType, num int) ([]dax.Address, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() + } + + workers, err := f.ListWorkers(tx, roleType) + if err != nil { + return nil, errors.Wrap(err, "listing free workers") + } + + if len(workers) < num { + return nil, errors.Errorf("not enough free workers to pop: wanted %d, have: %d", num, len(workers)) + } + + // Get num workers from the list. + workersToAssign := workers[0:num] + + // Create a workerset containing the free workers which remain after + // removing num workers. + workerset := dax.NewSet[dax.Address]() + for _, worker := range workers[num:] { + workerset.Add(worker) + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + val, err := encodeWorkerSet(workerset) + if err != nil { + return nil, errors.Wrap(err, "encoding worker set") + } + + if err := bkt.Put(freeWorkerKey(roleType), val); err != nil { + return nil, errors.Wrap(err, "putting free worker") + } + + return workersToAssign, nil +} + +func (f *freeWorkerService) ListWorkers(tx dax.Transaction, roleType dax.RoleType) (dax.Addresses, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() + } + + bkt := txx.Bucket(bucketBalancer) + if bkt == nil { + return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketBalancer) + } + + workerset := dax.NewSet[dax.Address]() + var err error + + // get free workers + fws := bkt.Get(freeWorkerKey(roleType)) + if fws != nil { + workerset, err = decodeWorkerSet(fws) + if err != nil { + return nil, errors.Wrap(err, "decoding worker set") + } + } + + return workerset.Sorted(), nil +} + +////////////////////////////////////////////////////// + +const ( + prefixFmtWorkersDB = "workers/role/%s/db/%s/" // %s - role, dbKey + prefixFmtWorkersAssigned = "workers/assigned/" + + prefixFmtFreeJobs = "freejobs/role/%s/db/%s" // %s - role, dbKey + prefixFmtFreeWorkers = "freeworkers/role/%s" // %s - role +) + +// workerDBKey returns a key based on worker. +// +// Format: workers/role/[role]/db/[dbKey]/[worker] = [job1, job2, ...] +func workerDBKey(roleType dax.RoleType, qdbid dax.QualifiedDatabaseID, addr dax.Address) []byte { + key := fmt.Sprintf(prefixFmtWorkersDB+"%s", roleType, qdbid.Key(), addr) + return []byte(key) +} + +// workerAssignedKey returns a key based on worker. +// +// Format: workers/assigned/[worker] = dbKey +func workerAssignedKey(addr dax.Address) []byte { + key := fmt.Sprintf(prefixFmtWorkersAssigned+"%s", addr) + return []byte(key) +} + +// keyWorker gets the worker out of the key. +func keyWorker(key []byte) (dax.Address, error) { + parts := strings.SplitN(string(key), "/", 6) + if len(parts) != 6 { + return "", errors.New(errors.ErrUncoded, "worker key format expected: `workers/role/[role]/db/[db]/worker`") + } + + return dax.Address(parts[5]), nil +} + +// freeJobKey returns a key for all freeJobs. +// +// Format: freejobs/role/[role]/db/[dbKey] = [job1, job2, ...] +func freeJobKey(roleType dax.RoleType, qdbid dax.QualifiedDatabaseID) []byte { + key := fmt.Sprintf(prefixFmtFreeJobs, roleType, qdbid.Key()) + return []byte(key) +} + +// freeWorkerKey returns a key for all freeWorkers. +// +// Format: freeworkers/role/[role] = [worker1, worker2, ...] +func freeWorkerKey(roleType dax.RoleType) []byte { + key := fmt.Sprintf(prefixFmtFreeWorkers, roleType) + return []byte(key) +} diff --git a/dax/mds/controller/naive/types.go b/dax/mds/controller/balancer/types.go similarity index 74% rename from dax/mds/controller/naive/types.go rename to dax/mds/controller/balancer/types.go index 6659b7f9d..90fcafa6e 100644 --- a/dax/mds/controller/naive/types.go +++ b/dax/mds/controller/balancer/types.go @@ -1,4 +1,4 @@ -package naive +package balancer import ( "sort" @@ -20,36 +20,36 @@ func newJobSetDiffs() jobSetDiffs { } } -type InternalDiffs map[dax.Worker]jobSetDiffs +type InternalDiffs map[dax.Address]jobSetDiffs func NewInternalDiffs() InternalDiffs { return make(InternalDiffs) } -func (d InternalDiffs) Added(worker dax.Worker, job dax.Job) { - if _, ok := d[worker]; !ok { - d[worker] = newJobSetDiffs() +func (d InternalDiffs) Added(address dax.Address, job dax.Job) { + if _, ok := d[address]; !ok { + d[address] = newJobSetDiffs() } // Before adding the job, make sure we haven't indicated that it has been // removed prior to this. If it has, we need to invalidate that "remove" // instruction. - d[worker].removed.Remove(job) + d[address].removed.Remove(job) - d[worker].added.Add(job) + d[address].added.Add(job) } -func (d InternalDiffs) Removed(worker dax.Worker, job dax.Job) { - if _, ok := d[worker]; !ok { - d[worker] = newJobSetDiffs() +func (d InternalDiffs) Removed(address dax.Address, job dax.Job) { + if _, ok := d[address]; !ok { + d[address] = newJobSetDiffs() } // Before removing the job, make sure we haven't indicated that it has been // added prior to this. If it has, we need to invalidate that "add" // instruction. - d[worker].added.Remove(job) + d[address].added.Remove(job) - d[worker].removed.Add(job) + d[address].removed.Add(job) } func (d InternalDiffs) Merge(d2 InternalDiffs) { @@ -69,7 +69,7 @@ func (d InternalDiffs) Output() []dax.WorkerDiff { i := 0 for k, v := range d { - out[i].WorkerID = k + out[i].Address = k out[i].AddedJobs = v.added.Sorted() out[i].RemovedJobs = v.removed.Sorted() i++ diff --git a/dax/mds/controller/config.go b/dax/mds/controller/config.go index 1d77fc73c..e67d3ddb0 100644 --- a/dax/mds/controller/config.go +++ b/dax/mds/controller/config.go @@ -11,10 +11,10 @@ import ( type NewBalancerFn func(string, logger.Logger) Balancer type Config struct { - Director Director - Schemar schemar.Schemar - ComputeBalancer Balancer - TranslateBalancer Balancer + Director Director + Schemar schemar.Schemar + + Balancer Balancer StorageMethod string BoltDB *boltdb.DB diff --git a/dax/mds/controller/controller.go b/dax/mds/controller/controller.go index df71b5bda..abb4332f3 100644 --- a/dax/mds/controller/controller.go +++ b/dax/mds/controller/controller.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "sort" - "sync" "time" "github.com/featurebasedb/featurebase/v3/dax" @@ -17,22 +16,17 @@ import ( ) type Controller struct { - // mu is primarily to protect against conflicting reads/writes to the nodes - // map. The balancers map is currently never written to after - // initialization. - mu sync.RWMutex - // Schemar used by the controller to get table information. The controller // should NOT call Schemar methods which modify data. Schema mutations are // made outside of the controller (at this point that happens in MDS). Schemar schemar.Schemar - // nodes is the map of nodes, by address, which have registered with the - // controller. + // nodeService is the interface to working with nodes, by address, which + // have registered with the controller. nodeService dax.NodeService - ComputeBalancer Balancer - TranslateBalancer Balancer + boltDB *boltdb.DB + Balancer Balancer // Director is used to send directives to computer workers. Director Director @@ -61,8 +55,8 @@ func New(cfg Config) *Controller { c := &Controller{ Schemar: schemar.NewNopSchemar(), - ComputeBalancer: cfg.ComputeBalancer, - TranslateBalancer: cfg.TranslateBalancer, + boltDB: cfg.BoltDB, + Balancer: cfg.Balancer, Director: NewNopDirector(), @@ -119,24 +113,16 @@ func (c *Controller) Stop() { close(c.stopping) } -func (c *Controller) balancerForRole(rt dax.RoleType) (Balancer, error) { - var bal Balancer - if rt == dax.RoleTypeCompute { - bal = c.ComputeBalancer - } else if rt == dax.RoleTypeTranslate { - bal = c.TranslateBalancer - } else { - return nil, errors.Errorf("unknown role type: '%s'", rt) - } - return bal, nil -} - // RegisterNodes adds nodes to the controller's list of registered // nodes. func (c *Controller) RegisterNodes(ctx context.Context, nodes ...*dax.Node) error { - c.logger.Printf("c.RegisterNodes(): %+v", nodes) - c.mu.Lock() - defer c.mu.Unlock() + c.logger.Printf("c.RegisterNodes(): %s", dax.Nodes(nodes)) + + tx, err := c.boltDB.BeginTx(ctx, true) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() // Validate input. for _, n := range nodes { @@ -159,8 +145,8 @@ func (c *Controller) RegisterNodes(ctx context.Context, nodes ...*dax.Node) erro // Create node if we don't already have it for _, n := range nodes { - if node, _ := c.nodeService.ReadNode(ctx, n.Address); node == nil { - if err := c.nodeService.CreateNode(ctx, n.Address, n); err != nil { + if node, _ := c.nodeService.ReadNode(tx, n.Address); node == nil { + if err := c.nodeService.CreateNode(tx, n.Address, n); err != nil { return errors.Wrapf(err, "creating node: %s", n.Address) } @@ -175,33 +161,24 @@ func (c *Controller) RegisterNodes(ctx context.Context, nodes ...*dax.Node) erro // diffByAddr keeps track of the diffs that have been applied to each // specific address. + // TODO(tlt): I don't understand why we're keeping track of the + // dax.WorkerDiff here (as opposed to just the unique Address) because it + // doesn't ever seem to be used. diffByAddr := make(map[dax.Address]dax.WorkerDiff) for _, n := range nodes { - for _, rt := range n.RoleTypes { - balancer, err := c.balancerForRole(rt) - if err != nil { - return errors.Wrap(err, "getting balancer") - } - adiffs, err := balancer.AddWorker(ctx, n.Address) - if err != nil { - return errors.Wrap(err, "adding worker") - } + adiffs, err := c.Balancer.AddWorker(tx, n) + if err != nil { + return errors.Wrap(err, "adding worker") + } - // Rebalance so existing jobs can be spread evenly across all nodes, - // including the node being registered. - bdiffs, err := balancer.Balance(ctx) - if err != nil { - return errors.Wrap(err, "balancing") - } - for _, diff := range append(adiffs, bdiffs...) { - existingDiff, ok := diffByAddr[dax.Address(diff.WorkerID)] - if !ok { - existingDiff.WorkerID = diff.WorkerID - } - existingDiff.Add(diff) - diffByAddr[dax.Address(diff.WorkerID)] = existingDiff + for _, diff := range adiffs { + existingDiff, ok := diffByAddr[dax.Address(diff.Address)] + if !ok { + existingDiff.Address = diff.Address } + existingDiff.Add(diff) + diffByAddr[dax.Address(diff.Address)] = existingDiff } } @@ -217,7 +194,7 @@ func (c *Controller) RegisterNodes(ctx context.Context, nodes ...*dax.Node) erro } // Tell the poller about the new nodes. - if err := c.poller.AddAddresses(ctx, addrs...); err != nil { + if err := c.poller.AddAddresses(tx.Context(), addrs...); err != nil { return NewErrInternal(err.Error()) } @@ -241,20 +218,17 @@ func (c *Controller) RegisterNodes(ctx context.Context, nodes ...*dax.Node) erro // Get the current job assignments for this worker and send that to the node // as a Directive. - if err := c.sendDirectives(ctx, addressMethods...); err != nil { + if err := c.sendDirectives(tx, addressMethods...); err != nil { return NewErrDirectiveSendFailure(err.Error()) } - return nil + return tx.Commit() } // RegisterNode adds a node to the controller's list of registered // nodes. It makes no guarantees about when the node will actually be // used for anything or assigned any jobs. func (c *Controller) RegisterNode(ctx context.Context, n *dax.Node) error { - c.mu.Lock() - defer c.mu.Unlock() - // Validate input. if n.Address == "" { return NewErrNodeKeyInvalid(n.Address) @@ -268,7 +242,13 @@ func (c *Controller) RegisterNode(ctx context.Context, n *dax.Node) error { } } - if node, _ := c.nodeService.ReadNode(ctx, n.Address); node != nil { + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + if node, _ := c.nodeService.ReadNode(tx, n.Address); node != nil { return nil } @@ -284,15 +264,18 @@ func (c *Controller) RegisterNode(ctx context.Context, n *dax.Node) error { // from its list (perhaps due to a network fault) and therefore the node needs // to be re-registered. func (c *Controller) CheckInNode(ctx context.Context, n *dax.Node) error { - c.mu.RLock() - defer c.mu.RUnlock() + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() // If we already know about this node, just no-op. In the future, we may // want this check-in payload to include things like the compute node's // Directive; then we could check that the compute node is actually doing // what we expect it to be doing. But for now, we're just checking that we // know about the compute node at all. - if node, _ := c.nodeService.ReadNode(ctx, n.Address); node != nil { + if node, _ := c.nodeService.ReadNode(tx, n.Address); node != nil { return nil } @@ -304,8 +287,11 @@ func (c *Controller) CheckInNode(ctx context.Context, n *dax.Node) error { // DeregisterNodes removes nodes from the controller's list of registered nodes. // It sends directives to the removed nodes, but ignores errors. func (c *Controller) DeregisterNodes(ctx context.Context, addresses ...dax.Address) error { - c.mu.Lock() - defer c.mu.Unlock() + tx, err := c.boltDB.BeginTx(ctx, true) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() // workerSet maintains the set of workers which have a job assignment change // and therefore need to be sent an updated Directive. @@ -325,40 +311,21 @@ func (c *Controller) DeregisterNodes(ctx context.Context, addresses ...dax.Addre // address this confusion. // workerSet.Add(address) - // Ensure the host:port is currently registered. - n, err := c.nodeService.ReadNode(ctx, address) + rdiffs, err := c.Balancer.RemoveWorker(tx, address) if err != nil { - return errors.Wrapf(err, "reading the node for address: %s", address) + return errors.Wrapf(err, "removing worker: %s", address) } - for _, rt := range n.RoleTypes { - balancer, err := c.balancerForRole(rt) - if err != nil { - c.logger.Printf("Unsupported role type in DeregisterNode: '%s'", rt) - // Skip any role types which aren't currently supported by a balancer. - continue - } - rdiffs, err := balancer.RemoveWorker(ctx, address) - if err != nil { - return errors.Wrap(err, "removing worker") - } - // Rebalance so any jobs that were assigned to the node being deregistered - // get assigned to another node. - bdiffs, err := balancer.Balance(ctx) - if err != nil { - return errors.Wrap(err, "balancing") - } - // we assume that the job names are different between the - // different role types so we don't have to track each - // role separately which would be annoying. - for _, diff := range append(rdiffs, bdiffs...) { - existingDiff, ok := diffByAddr[dax.Address(diff.WorkerID)] - if !ok { - existingDiff.WorkerID = diff.WorkerID - } - existingDiff.Add(diff) - diffByAddr[dax.Address(diff.WorkerID)] = existingDiff + // we assume that the job names are different between the + // different role types so we don't have to track each + // role separately which would be annoying. + for _, diff := range rdiffs { + existingDiff, ok := diffByAddr[dax.Address(diff.Address)] + if !ok { + existingDiff.Address = diff.Address } + existingDiff.Add(diff) + diffByAddr[dax.Address(diff.Address)] = existingDiff } } @@ -374,18 +341,18 @@ func (c *Controller) DeregisterNodes(ctx context.Context, addresses ...dax.Addre } for _, address := range addresses { - if err := c.nodeService.DeleteNode(ctx, address); err != nil { + if err := c.nodeService.DeleteNode(tx, address); err != nil { return errors.Wrapf(err, "deleting node at address: %s", address) } } - if err := c.poller.RemoveAddresses(ctx, addresses...); err != nil { + if err := c.poller.RemoveAddresses(tx.Context(), addresses...); err != nil { return NewErrInternal(err.Error()) } // No need to send Directives if nothing has ultimately changed. if len(workerSet) == 0 { - return nil + return tx.Commit() } // Convert the slice of addresses into a slice of addressMethod containing @@ -403,234 +370,92 @@ func (c *Controller) DeregisterNodes(ctx context.Context, addresses ...dax.Addre // Get the current job assignments for these workers and send them to the // nodes as Directives. - if err := c.sendDirectives(ctx, addressMethods...); err != nil { + if err := c.sendDirectives(tx, addressMethods...); err != nil { return NewErrDirectiveSendFailure(err.Error()) } - return nil + return tx.Commit() } -// Nodes returns the list of assigned nodes responsible for the jobs included in -// the given role. If createMissing is true, the Controller will create new jobs -// for any of which it isn't currently aware. -func (c *Controller) Nodes(ctx context.Context, role dax.Role, createMissing bool) ([]dax.AssignedNode, error) { - nodes := []dax.AssignedNode{} - var err error +func (c *Controller) nodesTranslateReadOrWrite(tx dax.Transaction, role *dax.TranslateRole, qdbid dax.QualifiedDatabaseID, createMissing bool, asWrite bool) ([]dax.AssignedNode, bool, error) { + qtid := role.TableKey.QualifiedTableID() + roleType := dax.RoleTypeTranslate - switch v := role.(type) { - case *dax.ComputeRole: - nodes, err = c.nodesCompute(ctx, v, createMissing) - if err != nil { - return nil, errors.Wrap(err, "getting compute nodes") - } - - case *dax.TranslateRole: - nodes, err = c.nodesTranslate(ctx, v, createMissing) - if err != nil { - return nil, errors.Wrap(err, "getting translate nodes") - } - } - - return nodes, nil -} - -// nodesTranslate is like nodesCompute. See the comments there. -func (c *Controller) nodesTranslate(ctx context.Context, role *dax.TranslateRole, createMissing bool) ([]dax.AssignedNode, error) { - // Try calling c.nodesTranslate as a read first. If we don't have to actually - // create any missing partitions, then we won't have to obtain a write lock. - translateNodes, retryAsWrite, err := c.nodesTranslateReadOrWrite(ctx, role, createMissing, false) - if err != nil { - return nil, errors.Wrap(err, "getting translate nodes read or write") - } - - if retryAsWrite { - translateNodes, _, err = c.nodesTranslateReadOrWrite(ctx, role, createMissing, retryAsWrite) - if err != nil { - return nil, errors.Wrap(err, "getting translate nodes read or write retry") - } - } - - return translateNodes, nil -} - -func (c *Controller) nodesTranslateReadOrWrite(ctx context.Context, role *dax.TranslateRole, createMissing bool, asWrite bool) ([]dax.AssignedNode, bool, error) { - if asWrite { - c.mu.Lock() - defer c.mu.Unlock() - } else { - c.mu.RLock() - defer c.mu.RUnlock() - } - - nodes := []dax.AssignedNode{} - - bal := c.TranslateBalancer - - //inJobs := NewStringSet() inJobs := dax.NewSet[dax.Job]() for _, p := range role.Partitions { partitionString := partition(role.TableKey, p).String() inJobs.Add(dax.Job(partitionString)) } - workers, err := bal.WorkersForJobs(ctx, inJobs.Sorted()) + workers, err := c.Balancer.WorkersForJobs(tx, roleType, qdbid, inJobs.Sorted()...) if err != nil { return nil, false, errors.Wrap(err, "getting workers for jobs") } - if createMissing { - // If any provided jobs were not returned in the WorkersForJobs - // request, then create those. - outJobs := dax.NewSet[dax.Job]() - for _, worker := range workers { - for _, job := range worker.Jobs { - outJobs.Add(job) - } - } - - missed := inJobs.Minus(outJobs).Sorted() - - if len(missed) > 0 { - // If we are currently under a read lock, and we get to this point, - // it means that we have partitions which need to be assigned (and - // directives sent) to workers. In that case, we need to abort this - // method run and notify the caller to rety as a write. - if !asWrite { - return nil, true, nil - } - - sort.Slice(missed, func(i, j int) bool { return missed[i] < missed[j] }) - - workerSet := NewAddressSet() - for _, job := range missed { - j, err := decodePartition(job) - if err != nil { - return nil, false, NewErrInternal(err.Error()) - } - diffs, err := bal.AddJobs(ctx, j) - if err != nil { - return nil, false, errors.Wrap(err, "adding job") - } - for _, diff := range diffs { - workerSet.Add(dax.Address(diff.WorkerID)) - } - } - - // Convert the slice of addresses into a slice of addressMethod containing - // the appropriate method. - addressMethods := applyAddressMethod(workerSet.SortedSlice(), dax.DirectiveMethodDiff) - - if err := c.sendDirectives(ctx, addressMethods...); err != nil { - return nil, false, NewErrDirectiveSendFailure(err.Error()) - } - - // Re-run WorkersForJobs. - workers, err = bal.WorkersForJobs(ctx, inJobs.Sorted()) - if err != nil { - return nil, false, errors.Wrap(err, "getting workers for jobs") - } + outJobs := dax.NewSet[dax.Job]() + for _, worker := range workers { + for _, job := range worker.Jobs { + outJobs.Add(job) } } - for _, worker := range workers { - // covert worker.Jobs []string to map[string][]Partition - translateMap := make(map[dax.TableKey]dax.PartitionNums) - for _, job := range worker.Jobs { + missed := inJobs.Minus(outJobs).Sorted() + if !createMissing && len(missed) > 0 { + return nil, false, NewErrUnassignedJobs(missed) + } + + // If any provided jobs were not returned in the WorkersForJobs request, + // then create those. + if createMissing && len(missed) > 0 { + // If we are currently under a read lock, and we get to this point, it + // means that we have partitions which need to be assigned (and + // directives sent) to workers. In that case, we need to abort this + // method run and notify the caller to rety as a write. + if !asWrite { + return nil, true, nil + } + + sort.Slice(missed, func(i, j int) bool { return missed[i] < missed[j] }) + + workerSet := NewAddressSet() + for _, job := range missed { j, err := decodePartition(job) if err != nil { return nil, false, NewErrInternal(err.Error()) } - - translateMap[j.table()] = append(translateMap[j.table()], j.partitionNum()) + diffs, err := c.Balancer.AddJobs(tx, roleType, qtid, j.Job()) + if err != nil { + return nil, false, errors.Wrap(err, "adding job") + } + for _, diff := range diffs { + workerSet.Add(dax.Address(diff.Address)) + } } - for table, partitions := range translateMap { - // Sort the partitions int slice before returning it. - sort.Sort(partitions) + // Convert the slice of addresses into a slice of addressMethod + // containing the appropriate method. + addressMethods := applyAddressMethod(workerSet.SortedSlice(), dax.DirectiveMethodDiff) - nodes = append(nodes, dax.AssignedNode{ - Address: dax.Address(worker.ID), - Role: &dax.TranslateRole{ - TableKey: table, - Partitions: partitions, - }, - }) + if err := c.sendDirectives(tx, addressMethods...); err != nil { + return nil, false, NewErrDirectiveSendFailure(err.Error()) } - } - return nodes, false, nil -} - -// nodesCompute tries to get the list of compute nodes under a read lock. If the -// call into c.nodesComputeReadOrWrite() comes back with `retryAsWrite = true`, -// then it gets called again but with a write lock so that sendDirective can -// happen, and the receiving compute node can get apply the latest schema, -// without encountering race conditions. -// -// Really, we shouldn't have to rely on the directive being applied within a -// mu.Lock(). Instead, if a client (or the IDK) tries to perform some action on -// a compute node (for example, ingesting data to an index/field), if that -// action fails because the schema on the compute node is not in sync, or if the -// compute node is completely unavailable, the client should ask mds for updated -// node information and keep retrying. Basically, what I'm saying is that a lot -// of the mu.Lock()s in this file can be changed back to mu.RLock()s, and the -// SendDirective() can happen asyncronously without worring about race -// conditions. -// -// The race condition happened when concurrent requests to Nodes() occurred and -// the order of events was: -// - req1 wants node for [idx, 0] -// - req2 wants node for [idx, 0] -// - (req1) [idx, 0] registered in controller to node A -// - directive sent to node A to create index [idx] -// -// - (req2) receives: node A -// - (req2) tries to ingest data to node A [idx, 0] -// **** RACE: [idx] does not exist because (req1) directive step is not compete -func (c *Controller) nodesCompute(ctx context.Context, role *dax.ComputeRole, createMissing bool) ([]dax.AssignedNode, error) { - if len(role.Shards) == 0 { - return c.nodesForTableKey(ctx, role.TableKey) - } - // Try calling c.nodesCompute as a read first. If we don't have to actually - // create any missing shards, then we won't have to obtain a write lock. - computeNodes, retryAsWrite, err := c.nodesComputeReadOrWrite(ctx, role, createMissing, false) - if err != nil { - return nil, errors.Wrap(err, "getting compute nodes read or write") - } - - if retryAsWrite { - computeNodes, _, err = c.nodesComputeReadOrWrite(ctx, role, createMissing, retryAsWrite) + // Re-run WorkersForJobs. + workers, err = c.Balancer.WorkersForJobs(tx, roleType, qdbid, inJobs.Sorted()...) if err != nil { - return nil, errors.Wrap(err, "getting compute nodes read or write retry") + return nil, false, errors.Wrap(err, "getting workers for jobs") } } - return computeNodes, nil -} - -func (c *Controller) nodesForTableKey(ctx context.Context, tk dax.TableKey) ([]dax.AssignedNode, error) { - bal := c.ComputeBalancer - workers, err := bal.WorkersForJobPrefix(ctx, string(tk)) - if err != nil { - return nil, errors.Wrapf(err, "getting workers for table: '%s'", tk) - } - - return c.workersToAssignedNodes(ctx, workers) - + nodes, err := c.translateWorkersToAssignedNodes(tx, workers) + return nodes, false, errors.Wrap(err, "converting to assigned nodes") } // nodesComputeReadOrWrite contains the logic for the c.nodesCompute() method, // but it supports being called with either a read or write lock. -func (c *Controller) nodesComputeReadOrWrite(ctx context.Context, role *dax.ComputeRole, createMissing bool, asWrite bool) ([]dax.AssignedNode, bool, error) { - if asWrite { - c.mu.Lock() - defer c.mu.Unlock() - } else { - c.mu.RLock() - defer c.mu.RUnlock() - } - - bal := c.ComputeBalancer +func (c *Controller) nodesComputeReadOrWrite(tx dax.Transaction, role *dax.ComputeRole, qdbid dax.QualifiedDatabaseID, createMissing bool, asWrite bool) ([]dax.AssignedNode, bool, error) { + qtid := role.TableKey.QualifiedTableID() + roleType := dax.RoleTypeCompute inJobs := dax.NewSet[dax.Job]() for _, s := range role.Shards { @@ -638,7 +463,7 @@ func (c *Controller) nodesComputeReadOrWrite(ctx context.Context, role *dax.Comp inJobs.Add(dax.Job(shardString)) } - workers, err := bal.WorkersForJobs(ctx, inJobs.Sorted()) + workers, err := c.Balancer.WorkersForJobs(tx, roleType, qdbid, inJobs.Sorted()...) if err != nil { return nil, false, errors.Wrap(err, "getting workers for jobs") } @@ -656,56 +481,54 @@ func (c *Controller) nodesComputeReadOrWrite(ctx context.Context, role *dax.Comp return nil, false, NewErrUnassignedJobs(missed) } - if createMissing { - // If any provided jobs were not returned in the WorkersForJobs - // request, then create those. - if len(missed) > 0 { - // If we are currently under a read lock, and we get to this point, - // it means that we have shards which need to be assigned (and - // directives sent) to workers. In that case, we need to abort this - // method run and notify the caller to rety as a write. - if !asWrite { - return nil, true, nil - } + // If any provided jobs were not returned in the WorkersForJobs request, + // then create those. + if createMissing && len(missed) > 0 { + // If we are currently under a read lock, and we get to this point, it + // means that we have shards which need to be assigned (and directives + // sent) to workers. In that case, we need to abort this method run and + // notify the caller to rety as a write. + if !asWrite { + return nil, true, nil + } - sort.Slice(missed, func(i, j int) bool { return missed[i] < missed[j] }) + sort.Slice(missed, func(i, j int) bool { return missed[i] < missed[j] }) - workerSet := NewAddressSet() - for _, job := range missed { - j, err := decodeShard(job) - if err != nil { - return nil, false, NewErrInternal(err.Error()) - } - diffs, err := bal.AddJobs(ctx, j) - if err != nil { - return nil, false, errors.Wrap(err, "adding job") - } - for _, diff := range diffs { - workerSet.Add(dax.Address(diff.WorkerID)) - } - } - - // Convert the slice of addresses into a slice of addressMethod containing - // the appropriate method. - addressMethods := applyAddressMethod(workerSet.SortedSlice(), dax.DirectiveMethodDiff) - - if err := c.sendDirectives(ctx, addressMethods...); err != nil { - return nil, false, NewErrDirectiveSendFailure(err.Error()) - } - - // Re-run WorkersForJobs. - workers, err = bal.WorkersForJobs(ctx, inJobs.Sorted()) + workerSet := NewAddressSet() + for _, job := range missed { + j, err := decodeShard(job) if err != nil { - return nil, false, errors.Wrap(err, "getting workers for jobs") + return nil, false, NewErrInternal(err.Error()) } + diffs, err := c.Balancer.AddJobs(tx, roleType, qtid, j.Job()) + if err != nil { + return nil, false, errors.Wrap(err, "adding job") + } + for _, diff := range diffs { + workerSet.Add(dax.Address(diff.Address)) + } + } + + // Convert the slice of addresses into a slice of addressMethod + // containing the appropriate method. + addressMethods := applyAddressMethod(workerSet.SortedSlice(), dax.DirectiveMethodDiff) + + if err := c.sendDirectives(tx, addressMethods...); err != nil { + return nil, false, NewErrDirectiveSendFailure(err.Error()) + } + + // Re-run WorkersForJobs. + workers, err = c.Balancer.WorkersForJobs(tx, roleType, qdbid, inJobs.Sorted()...) + if err != nil { + return nil, false, errors.Wrap(err, "getting workers for jobs") } } - nodes, err := c.workersToAssignedNodes(ctx, workers) + nodes, err := c.computeWorkersToAssignedNodes(tx, workers) return nodes, false, errors.Wrap(err, "converting to assigned nodes") } -func (c *Controller) workersToAssignedNodes(ctx context.Context, workers []dax.WorkerInfo) ([]dax.AssignedNode, error) { +func (c *Controller) computeWorkersToAssignedNodes(tx dax.Transaction, workers []dax.WorkerInfo) ([]dax.AssignedNode, error) { nodes := []dax.AssignedNode{} for _, worker := range workers { // convert worker.Jobs []string to map[TableName][]Shard @@ -724,7 +547,7 @@ func (c *Controller) workersToAssignedNodes(ctx context.Context, workers []dax.W sort.Sort(shards) nodes = append(nodes, dax.AssignedNode{ - Address: dax.Address(worker.ID), + Address: dax.Address(worker.Address), Role: &dax.ComputeRole{ TableKey: table, Shards: shards, @@ -735,11 +558,100 @@ func (c *Controller) workersToAssignedNodes(ctx context.Context, workers []dax.W return nodes, nil } -// CreateTable adds a table to the versionStore and schemar, and then sends directives -// to all affected nodes based on the change. +func (c *Controller) translateWorkersToAssignedNodes(tx dax.Transaction, workers []dax.WorkerInfo) ([]dax.AssignedNode, error) { + nodes := []dax.AssignedNode{} + for _, worker := range workers { + // covert worker.Jobs []string to map[string][]Partition + translateMap := make(map[dax.TableKey]dax.PartitionNums) + for _, job := range worker.Jobs { + j, err := decodePartition(job) + if err != nil { + return nil, NewErrInternal(err.Error()) + } + + translateMap[j.table()] = append(translateMap[j.table()], j.partitionNum()) + } + + for table, partitions := range translateMap { + // Sort the partitions int slice before returning it. + sort.Sort(partitions) + + nodes = append(nodes, dax.AssignedNode{ + Address: dax.Address(worker.Address), + Role: &dax.TranslateRole{ + TableKey: table, + Partitions: partitions, + }, + }) + } + } + return nodes, nil +} + +// CreateDatabase adds a database to the schemar. +func (c *Controller) CreateDatabase(ctx context.Context, qdb *dax.QualifiedDatabase) error { + tx, err := c.boltDB.BeginTx(ctx, true) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + if err := c.Schemar.CreateDatabase(tx, qdb); err != nil { + return errors.Wrap(err, "creating database in schemar") + } + + return tx.Commit() +} + +// DatabaseByID returns the database for the given id. +func (c *Controller) DatabaseByID(ctx context.Context, qdbid dax.QualifiedDatabaseID) (*dax.QualifiedDatabase, error) { + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + return nil, errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + qdb, err := c.Schemar.DatabaseByID(tx, qdbid) + if err != nil { + return nil, errors.Wrap(err, "getting database from schemar") + } + + return qdb, nil +} + +// SetDatabaseOptions sets the options on the given database. +func (c *Controller) SetDatabaseOptions(ctx context.Context, qdbid dax.QualifiedDatabaseID, opts dax.DatabaseOptions) error { + tx, err := c.boltDB.BeginTx(ctx, true) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + if err := c.Schemar.SetDatabaseOptions(tx, qdbid, opts); err != nil { + return errors.Wrap(err, "setting database options") + } + + return tx.Commit() +} + +// CreateTable adds a table to the schemar, and then sends directives to all +// affected nodes based on the change. func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) error { - c.mu.Lock() - defer c.mu.Unlock() + tx, err := c.boltDB.BeginTx(ctx, true) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + // Create Table ID. + if _, err := qtbl.CreateID(); err != nil { + return errors.Wrap(err, "creating table ID") + } + + // Create the table in schemar. + if err := c.Schemar.CreateTable(tx, qtbl); err != nil { + return errors.Wrapf(err, "creating table: %s", qtbl) + } // If the table is keyed, add partitions to the balancer. if qtbl.StringKeys() { @@ -753,24 +665,24 @@ func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) partitionsToAdd[partitionNum] = dax.PartitionNum(partitionNum) } - stringers := make([]fmt.Stringer, 0, len(partitionsToAdd)) + jobs := make([]dax.Job, 0, len(partitionsToAdd)) for _, p := range partitionsToAdd { - stringers = append(stringers, partition(qtbl.Key(), p)) + jobs = append(jobs, partition(qtbl.Key(), p).Job()) } - diffs, err := c.TranslateBalancer.AddJobs(ctx, stringers...) + diffs, err := c.Balancer.AddJobs(tx, dax.RoleTypeTranslate, qtbl.QualifiedID(), jobs...) if err != nil { return errors.Wrap(err, "adding job") } for _, diff := range diffs { - workerSet.Add(dax.Address(diff.WorkerID)) + workerSet.Add(dax.Address(diff.Address)) } // Convert the slice of addresses into a slice of addressMethod containing // the appropriate method. addressMethods := applyAddressMethod(workerSet.SortedSlice(), dax.DirectiveMethodDiff) - if err := c.sendDirectives(ctx, addressMethods...); err != nil { + if err := c.sendDirectives(tx, addressMethods...); err != nil { return NewErrDirectiveSendFailure(err.Error()) } } @@ -790,34 +702,37 @@ func (c *Controller) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) // We don't currently use the returned diff, other than to determine // which worker was affected, because we send the full Directive // every time. - diffs, err := c.TranslateBalancer.AddJobs(ctx, partition(qtbl.Key(), p)) + diffs, err := c.Balancer.AddJobs(tx, dax.RoleTypeTranslate, qtbl.QualifiedID(), partition(qtbl.Key(), p).Job()) if err != nil { return errors.Wrap(err, "adding job") } for _, diff := range diffs { - workerSet.Add(dax.Address(diff.WorkerID)) + workerSet.Add(dax.Address(diff.Address)) } // Convert the slice of addresses into a slice of addressMethod containing // the appropriate method. addressMethods := applyAddressMethod(workerSet.SortedSlice(), dax.DirectiveMethodDiff) - if err := c.sendDirectives(ctx, addressMethods...); err != nil { + if err := c.sendDirectives(tx, addressMethods...); err != nil { return NewErrDirectiveSendFailure(err.Error()) } } - return nil + return tx.Commit() } // DropTable removes a table from the schema and sends directives to all affected // nodes based on the change. func (c *Controller) DropTable(ctx context.Context, qtid dax.QualifiedTableID) error { - c.mu.Lock() - defer c.mu.Unlock() + tx, err := c.boltDB.BeginTx(ctx, true) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() // Get the table from the schemar. - if _, err := c.Schemar.Table(ctx, qtid); err != nil { + if _, err := c.Schemar.Table(tx, qtid); err != nil { return errors.Wrapf(err, "table not in schemar: %s", qtid) } @@ -825,90 +740,70 @@ func (c *Controller) DropTable(ctx context.Context, qtid dax.QualifiedTableID) e // and therefore need to be sent an updated Directive. workerSet := NewAddressSet() - diffs, err := c.ComputeBalancer.RemoveJobs(ctx, string(qtid.Key())) + diffs, err := c.Balancer.RemoveJobs(tx, dax.RoleTypeCompute, qtid) if err != nil { - return errors.Wrap(err, "removing jobs") + return errors.Wrapf(err, "removing compute jobs for table: %s", qtid) } for _, diff := range diffs { - workerSet.Add(dax.Address(diff.WorkerID)) + workerSet.Add(dax.Address(diff.Address)) } - diffs, err = c.TranslateBalancer.RemoveJobs(ctx, string(qtid.Key())) + diffs, err = c.Balancer.RemoveJobs(tx, dax.RoleTypeTranslate, qtid) if err != nil { - return errors.Wrap(err, "removing job") + return errors.Wrapf(err, "removing translate jobs for table: %s", qtid) } for _, diff := range diffs { - workerSet.Add(dax.Address(diff.WorkerID)) + workerSet.Add(dax.Address(diff.Address)) } // Convert the slice of addresses into a slice of addressMethod containing // the appropriate method. addressMethods := applyAddressMethod(workerSet.SortedSlice(), dax.DirectiveMethodDiff) - if err := c.sendDirectives(ctx, addressMethods...); err != nil { + if err := c.sendDirectives(tx, addressMethods...); err != nil { return NewErrDirectiveSendFailure(err.Error()) } - return nil + // Remove table from schemar. + if err := c.Schemar.DropTable(tx, qtid); err != nil { + return errors.Wrapf(err, "dropping table from schemar: %s", qtid) + } + + return tx.Commit() } // Table returns a table by quaified table id. func (c *Controller) Table(ctx context.Context, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) { - c.mu.Lock() - defer c.mu.Unlock() + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + return nil, errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() // Get the table from the schemar. - return c.Schemar.Table(ctx, qtid) + return c.Schemar.Table(tx, qtid) } // Tables returns a list of tables by name. -func (c *Controller) Tables(ctx context.Context, qual dax.TableQualifier, ids ...dax.TableID) ([]*dax.QualifiedTable, error) { - c.mu.Lock() - defer c.mu.Unlock() +func (c *Controller) Tables(ctx context.Context, qdbid dax.QualifiedDatabaseID, ids ...dax.TableID) ([]*dax.QualifiedTable, error) { + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + return nil, errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() // Get the tables from the schemar. - return c.Schemar.Tables(ctx, qual, ids...) -} - -// AddShards registers the table/shard combinations with the controller and -// sends the necessary directive. -func (c *Controller) AddShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) error { - c.mu.Lock() - defer c.mu.Unlock() - - // workerSet maintains the set of workers which have a job assignment change - // and therefore need to be sent an updated Directive. - workerSet := NewAddressSet() - - for _, s := range shards { - // We don't currently use the returned diff, other than to determine - // which worker was affected, because we send the full Directive every - // time. - diffs, err := c.ComputeBalancer.AddJobs(ctx, shard(qtid.Key(), s)) - if err != nil { - return errors.Wrap(err, "adding job") - } - for _, diff := range diffs { - workerSet.Add(dax.Address(diff.WorkerID)) - } - } - - // Convert the slice of addresses into a slice of addressMethod containing - // the appropriate method. - addressMethods := applyAddressMethod(workerSet.SortedSlice(), dax.DirectiveMethodDiff) - - if err := c.sendDirectives(ctx, addressMethods...); err != nil { - return NewErrDirectiveSendFailure(err.Error()) - } - - return nil + return c.Schemar.Tables(tx, qdbid, ids...) } // RemoveShards deregisters the table/shard combinations with the controller and // sends the necessary directives. func (c *Controller) RemoveShards(ctx context.Context, qtid dax.QualifiedTableID, shards ...dax.ShardNum) error { - c.mu.Lock() - defer c.mu.Unlock() + tx, err := c.boltDB.BeginTx(ctx, true) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() // workerSet maintains the set of workers which have a job assignment change // and therefore need to be sent an updated Directive. @@ -918,12 +813,12 @@ func (c *Controller) RemoveShards(ctx context.Context, qtid dax.QualifiedTableID // We don't currently use the returned diff, other than to determine // which worker was affected, because we send the full Directive every // time. - diffs, err := c.ComputeBalancer.RemoveJob(ctx, shard(qtid.Key(), s)) + diffs, err := c.Balancer.RemoveJobs(tx, dax.RoleTypeCompute, qtid, shard(qtid.Key(), s).Job()) if err != nil { return errors.Wrap(err, "removing job") } for _, diff := range diffs { - workerSet.Add(dax.Address(diff.WorkerID)) + workerSet.Add(dax.Address(diff.Address)) } } @@ -931,22 +826,22 @@ func (c *Controller) RemoveShards(ctx context.Context, qtid dax.QualifiedTableID // the appropriate method. addressMethods := applyAddressMethod(workerSet.SortedSlice(), dax.DirectiveMethodDiff) - if err := c.sendDirectives(ctx, addressMethods...); err != nil { + if err := c.sendDirectives(tx, addressMethods...); err != nil { return NewErrDirectiveSendFailure(err.Error()) } - return nil + return tx.Commit() } // sendDirectives sends a directive (based on the current balancer state) to // each of the nodes provided. -func (c *Controller) sendDirectives(ctx context.Context, addrs ...addressMethod) error { +func (c *Controller) sendDirectives(tx dax.Transaction, addrs ...addressMethod) error { // If nodes is empty, return early. if len(addrs) == 0 { return nil } - directives, err := c.buildDirectives(ctx, addrs) + directives, err := c.buildDirectives(tx, addrs) if err != nil { return errors.Wrap(err, "building directives") } @@ -957,7 +852,7 @@ func (c *Controller) sendDirectives(ctx context.Context, addrs ...addressMethod) i := i dir := dir eg.Go(func() error { - errs[i] = c.Director.SendDirective(ctx, dir) + errs[i] = c.Director.SendDirective(tx.Context(), dir) if dir.IsEmpty() { errs[i] = nil } @@ -1016,11 +911,11 @@ func applyAddressMethod(addrs []dax.Address, method dax.DirectiveMethod) []addre // buildDirectives builds a list of directives for the given addrs (i.e. nodes) // using information (i.e. current state) from the balancers. -func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod) ([]*dax.Directive, error) { +func (c *Controller) buildDirectives(tx dax.Transaction, addrs []addressMethod) ([]*dax.Directive, error) { directives := make([]*dax.Directive, len(addrs)) for i, addressMethod := range addrs { - dVersion, err := c.directiveVersion.Increment(ctx, 1) + dVersion, err := c.directiveVersion.Increment(tx, 1) if err != nil { return nil, errors.Wrap(err, "incrementing directive version") } @@ -1053,11 +948,7 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod) ownsPartition0 := make(map[dax.TableKey]struct{}, 0) for _, rt := range supportedRoleTypes { - bal, err := c.balancerForRole(rt) - if err != nil { - return nil, errors.Wrap(err, "getting balancer") - } - w, err := bal.WorkerState(ctx, dax.Worker(addressMethod.address.String())) + w, err := c.Balancer.WorkerState(tx, rt, addressMethod.address) if err != nil { return nil, errors.Wrapf(err, "getting worker state: %s", addressMethod.address) } @@ -1130,7 +1021,7 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod) // everything that way for now. for tkey := range ownsPartition0 { qtid := tkey.QualifiedTableID() - table, err := c.Schemar.Table(ctx, qtid) + table, err := c.Schemar.Table(tx, qtid) if err != nil { return nil, errors.Wrapf(err, "getting table: %s", tkey) } @@ -1165,7 +1056,7 @@ func (c *Controller) buildDirectives(ctx context.Context, addrs []addressMethod) if len(tableSet) > 0 { dTables := make([]*dax.QualifiedTable, 0) for qual, tblIDs := range tableSet.QualifiedSortedSlice() { - qtbls, err := c.Schemar.Tables(ctx, qual, tblIDs...) + qtbls, err := c.Schemar.Tables(tx, qual, tblIDs...) if err != nil { return nil, errors.Wrapf(err, "getting directive tables for qual: %s", qual) } @@ -1194,7 +1085,13 @@ func (c *Controller) SetPoller(poller dax.AddressManager) { // This is useful in the case where MDS has restarted (or has been replaced) and // its poller is emtpy (i.e. it doesn't know about any nodes). func (c *Controller) InitializePoller(ctx context.Context) error { - nodes, err := c.nodeService.Nodes(context.Background()) + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + nodes, err := c.nodeService.Nodes(tx) if err != nil { return errors.Wrap(err, "initializing poller") } @@ -1219,10 +1116,21 @@ func (c *Controller) SnapshotTable(ctx context.Context, qtid dax.QualifiedTableI // snapshot that shard, then increment its shard version for logs written to the // WriteLogger. func (c *Controller) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTableID, shardNum dax.ShardNum) error { + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + return c.snapshotShardData(tx, qtid, shardNum) +} + +func (c *Controller) snapshotShardData(tx dax.Transaction, qtid dax.QualifiedTableID, shardNum dax.ShardNum) error { + qdbid := qtid.QualifiedDatabaseID + // Get the node responsible for the shard. - bal := c.ComputeBalancer - job := shard(qtid.Key(), shardNum) - workers, err := bal.WorkersForJobs(ctx, []dax.Job{dax.Job(job.String())}) + job := shard(qtid.Key(), shardNum).Job() + workers, err := c.Balancer.WorkersForJobs(tx, dax.RoleTypeCompute, qdbid, job) if err != nil { return errors.Wrapf(err, "getting workers for jobs: %s", job) } @@ -1231,7 +1139,7 @@ func (c *Controller) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTa return nil } - addr := dax.Address(workers[0].ID) + addr := dax.Address(workers[0].Address) // Send the node a snapshot request. req := &dax.SnapshotShardDataRequest{ @@ -1240,7 +1148,7 @@ func (c *Controller) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTa ShardNum: shardNum, } - if err := c.Director.SendSnapshotShardDataRequest(ctx, req); err != nil { + if err := c.Director.SendSnapshotShardDataRequest(tx.Context(), req); err != nil { return NewErrInternal(err.Error()) } @@ -1251,10 +1159,21 @@ func (c *Controller) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTa // partition to snapshot the table keys for that partition, then increment its // version for logs written to the WriteLogger. func (c *Controller) SnapshotTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) error { + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + return c.snapshotTableKeys(tx, qtid, partitionNum) +} + +func (c *Controller) snapshotTableKeys(tx dax.Transaction, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) error { + qdbid := qtid.QualifiedDatabaseID + // Get the node responsible for the partition. - bal := c.TranslateBalancer - job := partition(qtid.Key(), partitionNum) - workers, err := bal.WorkersForJobs(ctx, []dax.Job{dax.Job(job.String())}) + job := partition(qtid.Key(), partitionNum).Job() + workers, err := c.Balancer.WorkersForJobs(tx, dax.RoleTypeTranslate, qdbid, job) if err != nil { return errors.Wrapf(err, "getting workers for jobs: %s", job) } @@ -1263,7 +1182,7 @@ func (c *Controller) SnapshotTableKeys(ctx context.Context, qtid dax.QualifiedTa return nil } - addr := dax.Address(workers[0].ID) + addr := dax.Address(workers[0].Address) // Send the node a snapshot request. req := &dax.SnapshotTableKeysRequest{ @@ -1272,7 +1191,7 @@ func (c *Controller) SnapshotTableKeys(ctx context.Context, qtid dax.QualifiedTa PartitionNum: partitionNum, } - if err := c.Director.SendSnapshotTableKeysRequest(ctx, req); err != nil { + if err := c.Director.SendSnapshotTableKeysRequest(tx.Context(), req); err != nil { return NewErrInternal(err.Error()) } @@ -1283,13 +1202,24 @@ func (c *Controller) SnapshotTableKeys(ctx context.Context, qtid dax.QualifiedTa // to snapshot the keys for that field, then increment its version for logs // written to the WriteLogger. func (c *Controller) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, field dax.FieldName) error { + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + return c.snapshotFieldKeys(tx, qtid, field) +} + +func (c *Controller) snapshotFieldKeys(tx dax.Transaction, qtid dax.QualifiedTableID, field dax.FieldName) error { + qdbid := qtid.QualifiedDatabaseID + // Get the node responsible for the field. - bal := c.TranslateBalancer // Field translation is currently handled by partition 0. partitionNum := dax.PartitionNum(0) - job := partition(qtid.Key(), partitionNum) + job := partition(qtid.Key(), partitionNum).Job() - workers, err := bal.WorkersForJobs(ctx, []dax.Job{dax.Job(job.String())}) + workers, err := c.Balancer.WorkersForJobs(tx, dax.RoleTypeTranslate, qdbid, job) if err != nil { return errors.Wrapf(err, "getting workers for jobs: %s", job) } @@ -1298,7 +1228,7 @@ func (c *Controller) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTa return nil } - addr := dax.Address(workers[0].ID) + addr := dax.Address(workers[0].Address) // Send the node a snapshot request. req := &dax.SnapshotFieldKeysRequest{ @@ -1307,7 +1237,7 @@ func (c *Controller) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTa Field: field, } - if err := c.Director.SendSnapshotFieldKeysRequest(ctx, req); err != nil { + if err := c.Director.SendSnapshotFieldKeysRequest(tx.Context(), req); err != nil { return NewErrInternal(err.Error()) } @@ -1316,17 +1246,48 @@ func (c *Controller) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTa ///////////// -func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards dax.ShardNums, isWrite bool) ([]dax.ComputeNode, error) { - inRole := &dax.ComputeRole{ +// ComputeNodes returns the compute nodes for the given table/shards. It always +// uses a read transaction. The writable equivalent to this method is +// `IngestShard`. +func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shards dax.ShardNums) ([]dax.ComputeNode, error) { + role := &dax.ComputeRole{ TableKey: qtid.Key(), Shards: shards, } + qdbid := qtid.QualifiedDatabaseID - nodes, err := c.Nodes(ctx, inRole, isWrite) + tx, err := c.boltDB.BeginTx(ctx, false) if err != nil { - return nil, errors.Wrap(err, "getting compute nodes") + return nil, errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + // If no shards are provided, get the nodes responsible for the entire + // table. + if len(role.Shards) == 0 { + assignedNodes, err := c.nodesForTable(tx, dax.RoleTypeCompute, qtid) + if err != nil { + return nil, errors.Wrap(err, "getting nodes for table") + } + computeNodes, err := c.assignedToComputeNodes(assignedNodes) + if err != nil { + return nil, errors.Wrap(err, "converting assigned to compute nodes") + } + return computeNodes, nil } + assignedNodes, _, err := c.nodesComputeReadOrWrite(tx, role, qdbid, false, false) + if err != nil { + return nil, errors.Wrap(err, "getting compute nodes read or write") + } + + return c.assignedToComputeNodes(assignedNodes) +} + +// assignedToComputeNodes converts the provided []dax.AssignedNode to +// []dax.ComputeNode. If any of the assigned nodes are not for RoleType +// "compute", an error will be returned. +func (c *Controller) assignedToComputeNodes(nodes []dax.AssignedNode) ([]dax.ComputeNode, error) { computeNodes := make([]dax.ComputeNode, 0) for _, node := range nodes { @@ -1347,17 +1308,63 @@ func (c *Controller) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID return computeNodes, nil } -func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions dax.PartitionNums, isWrite bool) ([]dax.TranslateNode, error) { - inRole := &dax.TranslateRole{ +// TranslateNodes returns the translate nodes for the given table/partitions. It +// always uses a read transaction. The writable equivalent to this method is +// `IngestPartition`. +func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitions dax.PartitionNums) ([]dax.TranslateNode, error) { + role := &dax.TranslateRole{ TableKey: qtid.Key(), Partitions: partitions, } + qdbid := qtid.QualifiedDatabaseID - nodes, err := c.Nodes(ctx, inRole, isWrite) + tx, err := c.boltDB.BeginTx(ctx, false) if err != nil { - return nil, errors.Wrap(err, "getting translate nodes") + return nil, errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + // If no partitions are provided, get the nodes responsible for the entire + // table. + if len(role.Partitions) == 0 { + assignedNodes, err := c.nodesForTable(tx, dax.RoleTypeTranslate, qtid) + if err != nil { + return nil, errors.Wrap(err, "getting nodes for table") + } + translateNodes, err := c.assignedToTranslateNodes(assignedNodes) + if err != nil { + return nil, errors.Wrap(err, "converting assigned to translate nodes") + } + return translateNodes, nil } + assignedNodes, _, err := c.nodesTranslateReadOrWrite(tx, role, qdbid, false, false) + if err != nil { + return nil, errors.Wrap(err, "getting translate nodes read or write") + } + + return c.assignedToTranslateNodes(assignedNodes) +} + +func (c *Controller) nodesForTable(tx dax.Transaction, roleType dax.RoleType, qtid dax.QualifiedTableID) ([]dax.AssignedNode, error) { + workers, err := c.Balancer.WorkersForTable(tx, roleType, qtid) + if err != nil { + return nil, errors.Wrapf(err, "getting workers for table: '%s'", qtid) + } + switch roleType { + case dax.RoleTypeCompute: + return c.computeWorkersToAssignedNodes(tx, workers) + case dax.RoleTypeTranslate: + return c.translateWorkersToAssignedNodes(tx, workers) + default: + return nil, errors.Errorf("unsupported role type: %s", roleType) + } +} + +// assignedToTranslateNodes converts the provided []dax.AssignedNode to +// []dax.TranslateNode. If any of the assigned nodes are not for RoleType +// "translate", an error will be returned. +func (c *Controller) assignedToTranslateNodes(nodes []dax.AssignedNode) ([]dax.TranslateNode, error) { translateNodes := make([]dax.TranslateNode, 0) for _, node := range nodes { @@ -1379,25 +1386,63 @@ func (c *Controller) TranslateNodes(ctx context.Context, qtid dax.QualifiedTable } func (c *Controller) IngestPartition(ctx context.Context, qtid dax.QualifiedTableID, partition dax.PartitionNum) (dax.Address, error) { - c.mu.RLock() - defer c.mu.RUnlock() + role := &dax.TranslateRole{ + TableKey: qtid.Key(), + Partitions: dax.PartitionNums{partition}, + } + qdbid := qtid.QualifiedDatabaseID - partitions := dax.PartitionNums{partition} - - nodes, err := c.TranslateNodes(ctx, qtid, partitions, true) + // Try with a read transaction first. + tx, err := c.boltDB.BeginTx(ctx, false) if err != nil { + return "", errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + if err := c.sanitizeQTID(tx, &qtid); err != nil { + return "", errors.Wrap(err, "sanitizing") + } + + // Verify that the table exists. + if _, err := c.Schemar.Table(tx, qtid); err != nil { return "", err } - if l := len(nodes); l == 0 { - return "", NewErrNoAvailableNode() - } else if l > 1 { - return "", - NewErrInternal( - fmt.Sprintf("unexpected number of nodes: %d", l)) + nodes, retryAsWrite, err := c.nodesTranslateReadOrWrite(tx, role, qdbid, true, false) + if err != nil { + return "", errors.Wrap(err, "getting translate nodes read or write") } - node := nodes[0] + // If it's writable, and we couldn't find all the partitions with just a + // read, try again with a write transaction. + if retryAsWrite { + tx.Rollback() + + tx, err = c.boltDB.BeginTx(ctx, true) + if err != nil { + return "", errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + nodes, _, err = c.nodesTranslateReadOrWrite(tx, role, qdbid, true, true) + if err != nil { + return "", errors.Wrap(err, "getting translate nodes read or write retry") + } + } + + translateNodes, err := c.assignedToTranslateNodes(nodes) + if err != nil { + return "", errors.Wrap(err, "converting assigned to translate nodes") + } + + if l := len(translateNodes); l == 0 { + return "", NewErrNoAvailableNode() + } else if l > 1 { + return "", NewErrInternal( + fmt.Sprintf("unexpected number of nodes: %d", l)) + } + + node := translateNodes[0] // Verify that the node returned is actually responsible for the partition // requested. @@ -1415,32 +1460,132 @@ func (c *Controller) IngestPartition(ctx context.Context, qtid dax.QualifiedTabl fmt.Sprintf("partition returned (%d) does not match requested (%d)", p, partition)) } + // Only commit if the transaction is writable. + if retryAsWrite { + return node.Address, tx.Commit() + } + + return node.Address, nil +} + +// IngestShard handles an ingest shard request. +func (c *Controller) IngestShard(ctx context.Context, qtid dax.QualifiedTableID, shrdNum dax.ShardNum) (dax.Address, error) { + role := &dax.ComputeRole{ + TableKey: qtid.Key(), + Shards: dax.ShardNums{shrdNum}, + } + qdbid := qtid.QualifiedDatabaseID + + // Try with a read transaction first. + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + return "", errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + if err := c.sanitizeQTID(tx, &qtid); err != nil { + return "", errors.Wrap(err, "sanitizing") + } + + // Verify that the table exists. + if _, err := c.Schemar.Table(tx, qtid); err != nil { + return "", err + } + + nodes, retryAsWrite, err := c.nodesComputeReadOrWrite(tx, role, qdbid, true, false) + if err != nil { + return "", errors.Wrap(err, "getting compute nodes read or write") + } + + // If it's writable, and we couldn't find all the partitions with just a + // read, try again with a write transaction. + if retryAsWrite { + tx.Rollback() + + tx, err = c.boltDB.BeginTx(ctx, true) + if err != nil { + return "", errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + nodes, _, err = c.nodesComputeReadOrWrite(tx, role, qdbid, true, true) + if err != nil { + return "", errors.Wrap(err, "getting compute nodes read or write retry") + } + } + + computeNodes, err := c.assignedToComputeNodes(nodes) + if err != nil { + return "", errors.Wrap(err, "converting assigned to compute nodes") + } + + if l := len(computeNodes); l == 0 { + return "", NewErrNoAvailableNode() + } else if l > 1 { + return "", NewErrInternal( + fmt.Sprintf("unexpected number of nodes: %d", l)) + } + + node := computeNodes[0] + + // Verify that the node returned is actually responsible for the shard + // requested. + if node.Table != qtid.Key() { + return "", NewErrInternal( + fmt.Sprintf("table returned (%s) does not match requested (%s)", node.Table, qtid)) + } else if l := len(node.Shards); l != 1 { + return "", NewErrInternal( + fmt.Sprintf("unexpected number of shards returned: %d", l)) + } else if s := node.Shards[0]; s != shrdNum { + return "", NewErrInternal( + fmt.Sprintf("shard returned (%d) does not match requested (%d)", s, shrdNum)) + } + + // Only commit if the transaction is writable. + if retryAsWrite { + return node.Address, tx.Commit() + } + return node.Address, nil } //// func (c *Controller) CreateField(ctx context.Context, qtid dax.QualifiedTableID, fld *dax.Field) error { - c.mu.Lock() - defer c.mu.Unlock() + tx, err := c.boltDB.BeginTx(ctx, true) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + if err := c.sanitizeQTID(tx, &qtid); err != nil { + return errors.Wrap(err, "sanitizing") + } + + // Create the field in schemar. + if err := c.Schemar.CreateField(tx, qtid, fld); err != nil { + return errors.Wrapf(err, "creating field: %s, %s", qtid, fld) + } // workerSet maintains the set of workers which have a job assignment change // and therefore need to be sent an updated Directive. workerSet := NewAddressSet() + qdbid := qtid.QualifiedDatabaseID + // Get the worker(s) responsible for partition 0. - job := partition(qtid.Key(), 0).String() - workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)}) + job := partition(qtid.Key(), 0).Job() + workers, err := c.Balancer.WorkersForJobs(tx, dax.RoleTypeTranslate, qdbid, job) if err != nil { return errors.Wrapf(err, "getting workers for job: %s", job) } for _, w := range workers { - workerSet.Add(dax.Address(w.ID)) + workerSet.Add(dax.Address(w.Address)) } // Get the list of workers responsible for shard data for this table. - if state, err := c.ComputeBalancer.CurrentState(ctx); err != nil { + if state, err := c.Balancer.CurrentState(tx, dax.RoleTypeCompute, qdbid); err != nil { return errors.Wrap(err, "getting current compute state") } else { for _, worker := range state { @@ -1448,7 +1593,7 @@ func (c *Controller) CreateField(ctx context.Context, qtid dax.QualifiedTableID, if shard, err := decodeShard(job); err != nil { return errors.Wrapf(err, "decoding shard: %s", job) } else if shard.table() == qtid.Key() { - workerSet.Add(dax.Address(worker.ID)) + workerSet.Add(dax.Address(worker.Address)) break } } @@ -1460,34 +1605,48 @@ func (c *Controller) CreateField(ctx context.Context, qtid dax.QualifiedTableID, addressMethods := applyAddressMethod(workerSet.SortedSlice(), dax.DirectiveMethodDiff) // Send a directive to any compute node responsible for this field. - if err := c.sendDirectives(ctx, addressMethods...); err != nil { + if err := c.sendDirectives(tx, addressMethods...); err != nil { return NewErrDirectiveSendFailure(err.Error()) } - return nil + return tx.Commit() } func (c *Controller) DropField(ctx context.Context, qtid dax.QualifiedTableID, fldName dax.FieldName) error { - c.mu.Lock() - defer c.mu.Unlock() + tx, err := c.boltDB.BeginTx(ctx, true) + if err != nil { + return errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + if err := c.sanitizeQTID(tx, &qtid); err != nil { + return errors.Wrap(err, "sanitizing") + } + + // Drop the field from schemar. + if err := c.Schemar.DropField(tx, qtid, fldName); err != nil { + return errors.Wrapf(err, "dropping field: %s, %s", qtid, fldName) + } // workerSet maintains the set of workers which have a job assignment change // and therefore need to be sent an updated Directive. workerSet := NewAddressSet() + qdbid := qtid.QualifiedDatabaseID + // Get the worker(s) responsible for partition 0. - job := partition(qtid.Key(), 0).String() - workers, err := c.TranslateBalancer.WorkersForJobs(ctx, []dax.Job{dax.Job(job)}) + job := partition(qtid.Key(), 0).Job() + workers, err := c.Balancer.WorkersForJobs(tx, dax.RoleTypeTranslate, qdbid, job) if err != nil { return errors.Wrapf(err, "getting workers for job: %s", job) } for _, w := range workers { - workerSet.Add(dax.Address(w.ID)) + workerSet.Add(dax.Address(w.Address)) } // Get the list of workers responsible for shard data for this table. - if state, err := c.ComputeBalancer.CurrentState(ctx); err != nil { + if state, err := c.Balancer.CurrentState(tx, dax.RoleTypeCompute, qdbid); err != nil { return errors.Wrap(err, "getting current compute state") } else { for _, worker := range state { @@ -1495,7 +1654,7 @@ func (c *Controller) DropField(ctx context.Context, qtid dax.QualifiedTableID, f if shard, err := decodeShard(job); err != nil { return errors.Wrapf(err, "decoding shard: %s", job) } else if shard.table() == qtid.Key() { - workerSet.Add(dax.Address(worker.ID)) + workerSet.Add(dax.Address(worker.Address)) break } } @@ -1507,11 +1666,11 @@ func (c *Controller) DropField(ctx context.Context, qtid dax.QualifiedTableID, f addressMethods := applyAddressMethod(workerSet.SortedSlice(), dax.DirectiveMethodDiff) // Send a directive to any compute node responsible for this field. - if err := c.sendDirectives(ctx, addressMethods...); err != nil { + if err := c.sendDirectives(tx, addressMethods...); err != nil { return NewErrDirectiveSendFailure(err.Error()) } - return nil + return tx.Commit() } ////////////////////////////////// @@ -1526,5 +1685,35 @@ func (c *Controller) RemoveAddresses(ctx context.Context, addrs ...dax.Address) } func (c *Controller) DebugNodes(ctx context.Context) ([]*dax.Node, error) { - return c.nodeService.Nodes(ctx) + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + return nil, errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + return c.nodeService.Nodes(tx) +} + +// sanitizeQTID populates Table.ID (by looking up the table, by name, in +// schemar) for a given table having only a Name value, but no ID. +func (c *Controller) sanitizeQTID(tx dax.Transaction, qtid *dax.QualifiedTableID) error { + if qtid.ID == "" { + nqtid, err := c.Schemar.TableID(tx, qtid.QualifiedDatabaseID, qtid.Name) + if err != nil { + return errors.Wrap(err, "getting table ID") + } + qtid.ID = nqtid.ID + } + return nil +} + +// TableID handles a table id (i.e. by name) request. +func (c *Controller) TableID(ctx context.Context, qdbid dax.QualifiedDatabaseID, name dax.TableName) (dax.QualifiedTableID, error) { + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + return dax.QualifiedTableID{}, errors.Wrap(err, "beginning tx") + } + defer tx.Rollback() + + return c.Schemar.TableID(tx, qdbid, name) } diff --git a/dax/mds/controller/controller_test.go b/dax/mds/controller/controller_test.go index e57148b16..d135738f5 100644 --- a/dax/mds/controller/controller_test.go +++ b/dax/mds/controller/controller_test.go @@ -19,16 +19,25 @@ import ( func TestController(t *testing.T) { ctx := context.Background() - qual := dax.NewTableQualifier("acme", "db1") + qdbid := dax.NewQualifiedDatabaseID("acme", "db1") t.Run("RegisterNode", func(t *testing.T) { director := newTestDirector() - schemar, cleanup := daxtest.NewSchemar(t) defer cleanup() + + db := testbolt.MustOpenDB(t) + db.InitializeBuckets(balancerdb.BalancerBuckets...) + db.InitializeBuckets(schemardb.SchemarBuckets...) + defer func() { + testbolt.MustCloseDB(t, db) + testbolt.CleanupDB(t, db.Path()) + }() + cfg := controller.Config{ Director: director, Schemar: schemar, + BoltDB: db, } con := controller.New(cfg) @@ -61,19 +70,19 @@ func TestController(t *testing.T) { defer cleanup() db := testbolt.MustOpenDB(t) - db.InitializeBuckets(boltdb.NaiveBalancerBuckets...) + db.InitializeBuckets(balancerdb.BalancerBuckets...) + db.InitializeBuckets(schemardb.SchemarBuckets...) defer func() { testbolt.MustCloseDB(t, db) testbolt.CleanupDB(t, db.Path()) }() cfg := controller.Config{ - Director: director, - Schemar: schemar, - BoltDB: db, - StorageMethod: "boltdb", - ComputeBalancer: boltdb.NewBalancer("compute", db, logger.StderrLogger), - TranslateBalancer: boltdb.NewBalancer("translate", db, logger.StderrLogger), + Director: director, + Schemar: schemar, + BoltDB: db, + StorageMethod: "boltdb", + Balancer: balancerdb.NewBalancer(db, schemar, logger.StderrLogger), } con := controller.New(cfg) @@ -100,16 +109,28 @@ func TestController(t *testing.T) { } assert.Equal(t, exp, director.flush()) + // Add a qualified database. + dbOptions := dax.DatabaseOptions{ + WorkersMin: 1, + WorkersMax: 1, + } + qdb1 := daxtest.TestQualifiedDatabaseWithID(t, qdbid.OrganizationID, qdbid.DatabaseID, "dbname1", dbOptions) + assert.NoError(t, con.CreateDatabase(ctx, qdb1)) + + // tbls keeps the sorted list of tables used in tests + var tbls dax.QualifiedTables + // Add a non-keyed table. - tbl0 := daxtest.TestQualifiedTableWithID(t, qual, "2", "foo", 0, false) - assert.NoError(t, schemar.CreateTable(ctx, tbl0)) + tbl0 := daxtest.TestQualifiedTable(t, qdbid, "foo", 0, false) assert.NoError(t, con.CreateTable(ctx, tbl0)) + tbls = append(tbls, tbl0) + exp = []*dax.Directive{} assert.Equal(t, exp, director.flush()) // Add a shard. - assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), 0)) + addShards(t, ctx, con, tbl0.QualifiedID(), 0) exp = []*dax.Directive{ { @@ -130,6 +151,13 @@ func TestController(t *testing.T) { } assert.Equal(t, exp, director.flush()) + // Set WorkersMin to 3 so we can used the added nodes that follow. + { + dbOptions.WorkersMin = 3 + dbOptions.WorkersMax = 3 + assert.NoError(t, con.SetDatabaseOptions(ctx, qdb1.QualifiedID(), dbOptions)) + } + // Register two more nodes. node1 := &dax.Node{ Address: "10.0.0.1:81", @@ -172,11 +200,11 @@ func TestController(t *testing.T) { assert.Equal(t, exp, director.flush()) // Add more shards. - assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), dax.NewShardNums(1, 2, 3, 5, 8)...)) + addShards(t, ctx, con, tbl0.QualifiedID(), dax.NewShardNums(1, 2, 3, 5, 8)...) exp = []*dax.Directive{ { - Address: node0.Address, + Address: node1.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ tbl0, @@ -184,14 +212,14 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl0.Key(), - Shards: dax.NewShardNums(0, 3), + Shards: dax.NewShardNums(1), }, }, TranslateRoles: []dax.TranslateRole{}, Version: 5, }, { - Address: node1.Address, + Address: node2.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ tbl0, @@ -199,99 +227,151 @@ func TestController(t *testing.T) { ComputeRoles: []dax.ComputeRole{ { TableKey: tbl0.Key(), - Shards: dax.NewShardNums(1, 5), + Shards: dax.NewShardNums(2), }, }, TranslateRoles: []dax.TranslateRole{}, Version: 6, }, - { - Address: node2.Address, - Method: dax.DirectiveMethodDiff, - Tables: []*dax.QualifiedTable{ - tbl0, - }, - ComputeRoles: []dax.ComputeRole{ - { - TableKey: tbl0.Key(), - Shards: dax.NewShardNums(2, 8), - }, - }, - TranslateRoles: []dax.TranslateRole{}, - Version: 7, - }, - } - assert.Equal(t, exp, director.flush()) - - // Add another non-keyed table. - tbl1 := daxtest.TestQualifiedTableWithID(t, qual, "1", "bar", 0, false) - assert.NoError(t, schemar.CreateTable(ctx, tbl1)) - assert.NoError(t, con.CreateTable(ctx, tbl1)) - - // Add more shards. - assert.NoError(t, con.AddShards(ctx, tbl1.QualifiedID(), dax.NewShardNums(3, 5, 8, 13)...)) - - exp = []*dax.Directive{ { Address: node0.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, tbl0, }, ComputeRoles: []dax.ComputeRole{ - { - TableKey: tbl1.Key(), - Shards: dax.NewShardNums(3, 13), - }, { TableKey: tbl0.Key(), Shards: dax.NewShardNums(0, 3), }, }, TranslateRoles: []dax.TranslateRole{}, - Version: 8, + Version: 7, }, { Address: node1.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, tbl0, }, ComputeRoles: []dax.ComputeRole{ - { - TableKey: tbl1.Key(), - Shards: dax.NewShardNums(5), - }, { TableKey: tbl0.Key(), Shards: dax.NewShardNums(1, 5), }, }, TranslateRoles: []dax.TranslateRole{}, - Version: 9, + Version: 8, }, { Address: node2.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, tbl0, }, ComputeRoles: []dax.ComputeRole{ - { - TableKey: tbl1.Key(), - Shards: dax.NewShardNums(8), - }, { TableKey: tbl0.Key(), Shards: dax.NewShardNums(2, 8), }, }, TranslateRoles: []dax.TranslateRole{}, + Version: 9, + }, + } + assert.Equal(t, exp, director.flush()) + + // Add another non-keyed table. + tbl1 := daxtest.TestQualifiedTable(t, qdbid, "bar", 0, false) + assert.NoError(t, con.CreateTable(ctx, tbl1)) + + tbls = append(tbls, tbl1) + sort.Sort(tbls) + + // Add more shards. + addShards(t, ctx, con, tbl1.QualifiedID(), dax.NewShardNums(3, 5, 8, 13)...) + + exp = []*dax.Directive{ + { + Address: node0.Address, + Method: dax.DirectiveMethodDiff, + Tables: []*dax.QualifiedTable{ + tbls[0], + tbls[1], + }, + ComputeRoles: []dax.ComputeRole{ + { + TableKey: tbls[0].Key(), + Shards: dax.NewShardNums(3), + }, + { + TableKey: tbls[1].Key(), + Shards: dax.NewShardNums(0, 3), + }, + }, + TranslateRoles: []dax.TranslateRole{}, Version: 10, }, + { + Address: node1.Address, + Method: dax.DirectiveMethodDiff, + Tables: []*dax.QualifiedTable{ + tbls[0], + tbls[1], + }, + ComputeRoles: []dax.ComputeRole{ + { + TableKey: tbls[0].Key(), + Shards: dax.NewShardNums(5), + }, + { + TableKey: tbls[1].Key(), + Shards: dax.NewShardNums(1, 5), + }, + }, + TranslateRoles: []dax.TranslateRole{}, + Version: 11, + }, + { + Address: node2.Address, + Method: dax.DirectiveMethodDiff, + Tables: []*dax.QualifiedTable{ + tbls[0], + tbls[1], + }, + ComputeRoles: []dax.ComputeRole{ + { + TableKey: tbls[0].Key(), + Shards: dax.NewShardNums(8), + }, + { + TableKey: tbls[1].Key(), + Shards: dax.NewShardNums(2, 8), + }, + }, + TranslateRoles: []dax.TranslateRole{}, + Version: 12, + }, + { + Address: node0.Address, + Method: dax.DirectiveMethodDiff, + Tables: []*dax.QualifiedTable{ + tbls[0], + tbls[1], + }, + ComputeRoles: []dax.ComputeRole{ + { + TableKey: tbls[0].Key(), + Shards: dax.NewShardNums(3, 13), + }, + { + TableKey: tbls[1].Key(), + Shards: dax.NewShardNums(0, 3), + }, + }, + TranslateRoles: []dax.TranslateRole{}, + Version: 13, + }, } assert.Equal(t, exp, director.flush()) @@ -303,41 +383,41 @@ func TestController(t *testing.T) { Address: node0.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, - tbl0, + tbls[0], + tbls[1], }, ComputeRoles: []dax.ComputeRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Shards: dax.NewShardNums(3, 13), }, { - TableKey: tbl0.Key(), + TableKey: tbls[1].Key(), Shards: dax.NewShardNums(0, 1, 3), }, }, TranslateRoles: []dax.TranslateRole{}, - Version: 11, + Version: 14, }, { Address: node2.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, - tbl0, + tbls[0], + tbls[1], }, ComputeRoles: []dax.ComputeRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Shards: dax.NewShardNums(5, 8), }, { - TableKey: tbl0.Key(), + TableKey: tbls[1].Key(), Shards: dax.NewShardNums(2, 5, 8), }, }, TranslateRoles: []dax.TranslateRole{}, - Version: 12, + Version: 15, }, } assert.Equal(t, exp, director.flush()) @@ -350,21 +430,21 @@ func TestController(t *testing.T) { Address: node2.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, - tbl0, + tbls[0], + tbls[1], }, ComputeRoles: []dax.ComputeRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Shards: dax.NewShardNums(3, 5, 8, 13), }, { - TableKey: tbl0.Key(), + TableKey: tbls[1].Key(), Shards: dax.NewShardNums(0, 1, 2, 3, 5, 8), }, }, TranslateRoles: []dax.TranslateRole{}, - Version: 13, + Version: 16, }, } assert.Equal(t, exp, director.flush()) @@ -375,6 +455,14 @@ func TestController(t *testing.T) { exp = []*dax.Directive{} assert.Equal(t, exp, director.flush()) + // Set WorkersMin to 1 so we can add a single node and have it be used + // (currently just adding 1 node won't satisfy the minimum of 3). + { + dbOptions.WorkersMin = 1 + dbOptions.WorkersMax = 1 + assert.NoError(t, con.SetDatabaseOptions(ctx, qdb1.QualifiedID(), dbOptions)) + } + // Add a new node and ensure that the free shards get assigned to it. node3 := &dax.Node{ Address: "10.0.0.1:83", @@ -389,21 +477,21 @@ func TestController(t *testing.T) { Address: node3.Address, Method: dax.DirectiveMethodReset, Tables: []*dax.QualifiedTable{ - tbl1, - tbl0, + tbls[0], + tbls[1], }, ComputeRoles: []dax.ComputeRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Shards: dax.NewShardNums(3, 5, 8, 13), }, { - TableKey: tbl0.Key(), + TableKey: tbls[1].Key(), Shards: dax.NewShardNums(0, 1, 2, 3, 5, 8), }, }, TranslateRoles: []dax.TranslateRole{}, - Version: 14, + Version: 17, }, } assert.Equal(t, exp, director.flush()) @@ -416,21 +504,21 @@ func TestController(t *testing.T) { Address: node3.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, - tbl0, + tbls[0], + tbls[1], }, ComputeRoles: []dax.ComputeRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Shards: dax.NewShardNums(3, 5, 8, 13), }, { - TableKey: tbl0.Key(), + TableKey: tbls[1].Key(), Shards: dax.NewShardNums(0, 1, 3, 8), }, }, TranslateRoles: []dax.TranslateRole{}, - Version: 15, + Version: 18, }, } assert.Equal(t, exp, director.flush()) @@ -445,21 +533,21 @@ func TestController(t *testing.T) { Address: node3.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, - tbl0, + tbls[0], + tbls[1], }, ComputeRoles: []dax.ComputeRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Shards: dax.NewShardNums(3, 5, 8, 13), }, { - TableKey: tbl0.Key(), + TableKey: tbls[1].Key(), Shards: dax.NewShardNums(0, 1, 8), }, }, TranslateRoles: []dax.TranslateRole{}, - Version: 16, + Version: 19, }, } assert.Equal(t, exp, director.flush()) @@ -472,31 +560,27 @@ func TestController(t *testing.T) { Address: node3.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, + tbls[0], }, ComputeRoles: []dax.ComputeRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Shards: dax.NewShardNums(3, 5, 8, 13), }, }, TranslateRoles: []dax.TranslateRole{}, - Version: 17, + Version: 20, }, } assert.Equal(t, exp, director.flush()) // Remove a node which doesn't exist. - err := con.DeregisterNodes(ctx, "invalidNode") - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrNodeDoesNotExist)) - } - + assert.NoError(t, con.DeregisterNodes(ctx, "invalidNode")) }) t.Run("TranslateNodes", func(t *testing.T) { invalidQtid := dax.NewQualifiedTableID( - dax.NewTableQualifier("", ""), + dax.NewQualifiedDatabaseID("", ""), dax.TableID("invalidID"), ) @@ -505,19 +589,19 @@ func TestController(t *testing.T) { defer cleanup() db := testbolt.MustOpenDB(t) - db.InitializeBuckets(boltdb.NaiveBalancerBuckets...) + db.InitializeBuckets(balancerdb.BalancerBuckets...) + db.InitializeBuckets(schemardb.SchemarBuckets...) defer func() { testbolt.MustCloseDB(t, db) testbolt.CleanupDB(t, db.Path()) }() cfg := controller.Config{ - Director: director, - Schemar: schemar, - BoltDB: db, - StorageMethod: "boltdb", - ComputeBalancer: boltdb.NewBalancer("compute", db, logger.StderrLogger), - TranslateBalancer: boltdb.NewBalancer("translate", db, logger.StderrLogger), + Director: director, + Schemar: schemar, + BoltDB: db, + StorageMethod: "boltdb", + Balancer: balancerdb.NewBalancer(db, schemar, logger.StderrLogger), } con := controller.New(cfg) @@ -550,11 +634,23 @@ func TestController(t *testing.T) { exp = []*dax.Directive{} assert.Equal(t, exp, director.flush()) + // Add a qualified database. + dbOptions := dax.DatabaseOptions{ + WorkersMin: 1, + WorkersMax: 1, + } + qdb1 := daxtest.TestQualifiedDatabaseWithID(t, qdbid.OrganizationID, qdbid.DatabaseID, "dbname1", dbOptions) + assert.NoError(t, con.CreateDatabase(ctx, qdb1)) + + // tbls keeps the sorted list of tables used in tests + var tbls dax.QualifiedTables + // Add a keyed table. - tbl0 := daxtest.TestQualifiedTableWithID(t, qual, "2", "foo", 8, true) - assert.NoError(t, schemar.CreateTable(ctx, tbl0)) + tbl0 := daxtest.TestQualifiedTable(t, qdbid, "foo", 8, true) assert.NoError(t, con.CreateTable(ctx, tbl0)) + tbls = append(tbls, tbl0) + // Check directives. exp = []*dax.Directive{ { @@ -575,6 +671,13 @@ func TestController(t *testing.T) { } assert.Equal(t, exp, director.flush()) + // Set WorkersMin to 3 so we can used the two added nodes that follow. + { + dbOptions.WorkersMin = 3 + dbOptions.WorkersMax = 3 + assert.NoError(t, con.SetDatabaseOptions(ctx, qdb1.QualifiedID(), dbOptions)) + } + // Register two more nodes. node1 := &dax.Node{ Address: "10.0.0.1:81", @@ -586,34 +689,12 @@ func TestController(t *testing.T) { exp = []*dax.Directive{ { - Address: node0.Address, - Method: dax.DirectiveMethodDiff, - Tables: []*dax.QualifiedTable{ - tbl0, - }, - ComputeRoles: []dax.ComputeRole{}, - TranslateRoles: []dax.TranslateRole{ - { - TableKey: tbl0.Key(), - Partitions: dax.NewPartitionNums(0, 1, 2, 3), - }, - }, - Version: 3, - }, - { - Address: node1.Address, - Method: dax.DirectiveMethodReset, - Tables: []*dax.QualifiedTable{ - tbl0, - }, - ComputeRoles: []dax.ComputeRole{}, - TranslateRoles: []dax.TranslateRole{ - { - TableKey: tbl0.Key(), - Partitions: dax.NewPartitionNums(4, 5, 6, 7), - }, - }, - Version: 4, + Address: node1.Address, + Method: dax.DirectiveMethodReset, + Tables: []*dax.QualifiedTable{}, + ComputeRoles: []dax.ComputeRole{}, + TranslateRoles: []dax.TranslateRole{}, + Version: 3, }, } assert.Equal(t, exp, director.flush()) @@ -640,7 +721,7 @@ func TestController(t *testing.T) { Partitions: dax.NewPartitionNums(0, 1, 2), }, }, - Version: 5, + Version: 4, }, { Address: node1.Address, @@ -652,10 +733,10 @@ func TestController(t *testing.T) { TranslateRoles: []dax.TranslateRole{ { TableKey: tbl0.Key(), - Partitions: dax.NewPartitionNums(4, 5, 6), + Partitions: dax.NewPartitionNums(3, 5, 7), }, }, - Version: 6, + Version: 5, }, { Address: node2.Address, @@ -667,10 +748,10 @@ func TestController(t *testing.T) { TranslateRoles: []dax.TranslateRole{ { TableKey: tbl0.Key(), - Partitions: dax.NewPartitionNums(3, 7), + Partitions: dax.NewPartitionNums(4, 6), }, }, - Version: 7, + Version: 6, }, } assert.Equal(t, exp, director.flush()) @@ -679,71 +760,73 @@ func TestController(t *testing.T) { // Make PartitionN double digit to ensure that partition ints aren't // sorted as strings. Also, it should be large enough to spill over // onto node0. - tbl1 := daxtest.TestQualifiedTableWithID(t, qual, "1", "bar", 24, true) - assert.NoError(t, schemar.CreateTable(ctx, tbl1)) + tbl1 := daxtest.TestQualifiedTable(t, qdbid, "bar", 24, true) assert.NoError(t, con.CreateTable(ctx, tbl1)) + tbls = append(tbls, tbl1) + sort.Sort(tbls) + // Check directives. exp = []*dax.Directive{ { Address: node0.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, - tbl0, + tbls[0], + tbls[1], }, ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Partitions: dax.NewPartitionNums(1, 4, 7, 10, 13, 16, 19, 22), }, { - TableKey: tbl0.Key(), + TableKey: tbls[1].Key(), Partitions: dax.NewPartitionNums(0, 1, 2), }, }, - Version: 8, + Version: 7, }, { Address: node1.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, - tbl0, + tbls[0], + tbls[1], }, ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Partitions: dax.NewPartitionNums(2, 5, 8, 11, 14, 17, 20, 23), }, { - TableKey: tbl0.Key(), - Partitions: dax.NewPartitionNums(4, 5, 6), + TableKey: tbls[1].Key(), + Partitions: dax.NewPartitionNums(3, 5, 7), }, }, - Version: 9, + Version: 8, }, { Address: node2.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, - tbl0, + tbls[0], + tbls[1], }, ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Partitions: dax.NewPartitionNums(0, 3, 6, 9, 12, 15, 18, 21), }, { - TableKey: tbl0.Key(), - Partitions: dax.NewPartitionNums(3, 7), + TableKey: tbls[1].Key(), + Partitions: dax.NewPartitionNums(4, 6), }, }, - Version: 10, + Version: 9, }, } assert.Equal(t, exp, director.flush()) @@ -757,46 +840,46 @@ func TestController(t *testing.T) { Address: node0.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, + tbls[0], }, ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Partitions: dax.NewPartitionNums(1, 4, 7, 10, 13, 16, 19, 22), }, }, - Version: 11, + Version: 10, }, { Address: node1.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, + tbls[0], }, ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Partitions: dax.NewPartitionNums(2, 5, 8, 11, 14, 17, 20, 23), }, }, - Version: 12, + Version: 11, }, { Address: node2.Address, Method: dax.DirectiveMethodDiff, Tables: []*dax.QualifiedTable{ - tbl1, + tbls[0], }, ComputeRoles: []dax.ComputeRole{}, TranslateRoles: []dax.TranslateRole{ { - TableKey: tbl1.Key(), + TableKey: tbls[0].Key(), Partitions: dax.NewPartitionNums(0, 3, 6, 9, 12, 15, 18, 21), }, }, - Version: 13, + Version: 12, }, } assert.Equal(t, exp, director.flush()) @@ -822,18 +905,18 @@ func TestController(t *testing.T) { defer cleanup() db := testbolt.MustOpenDB(t) - db.InitializeBuckets(boltdb.NaiveBalancerBuckets...) + db.InitializeBuckets(balancerdb.BalancerBuckets...) + db.InitializeBuckets(schemardb.SchemarBuckets...) defer func() { testbolt.MustCloseDB(t, db) testbolt.CleanupDB(t, db.Path()) }() cfg := controller.Config{ - Schemar: schemar, - BoltDB: db, - StorageMethod: "boltdb", - ComputeBalancer: boltdb.NewBalancer("compute", db, logger.StderrLogger), - TranslateBalancer: boltdb.NewBalancer("translate", db, logger.StderrLogger), + Schemar: schemar, + BoltDB: db, + StorageMethod: "boltdb", + Balancer: balancerdb.NewBalancer(db, schemar, logger.StderrLogger), } con := controller.New(cfg) @@ -855,39 +938,42 @@ func TestController(t *testing.T) { } assert.NoError(t, con.RegisterNodes(ctx, node1)) + // Add a qualified database. + dbOptions := dax.DatabaseOptions{ + WorkersMin: 2, + WorkersMax: 2, + } + qdb1 := daxtest.TestQualifiedDatabaseWithID(t, qdbid.OrganizationID, qdbid.DatabaseID, "dbname1", dbOptions) + assert.NoError(t, con.CreateDatabase(ctx, qdb1)) + // Add a keyed table. - tbl0 := daxtest.TestQualifiedTable(t, qual, "foo", 12, true) - assert.NoError(t, schemar.CreateTable(ctx, tbl0)) + tbl0 := daxtest.TestQualifiedTable(t, qdbid, "foo", 12, true) assert.NoError(t, con.CreateTable(ctx, tbl0)) // Add shards. - assert.NoError(t, con.AddShards(ctx, tbl0.QualifiedID(), 0, 1, 2, 3, 11, 12)) + addShards(t, ctx, con, tbl0.QualifiedID(), 0, 1, 2, 3, 11, 12) - t.Run("ComputeRole", func(t *testing.T) { + t.Run("ComputeNodes", func(t *testing.T) { tests := []struct { - role dax.Role + role *dax.ComputeRole isWrite bool - exp []dax.AssignedNode + exp []dax.ComputeNode }{ { role: &dax.ComputeRole{ TableKey: tbl0.Key(), Shards: dax.NewShardNums(0, 1, 2, 3), }, - exp: []dax.AssignedNode{ + exp: []dax.ComputeNode{ { Address: node0.Address, - Role: &dax.ComputeRole{ - TableKey: tbl0.Key(), - Shards: dax.NewShardNums(0, 2), - }, + Table: tbl0.Key(), + Shards: dax.NewShardNums(0, 2), }, { Address: node1.Address, - Role: &dax.ComputeRole{ - TableKey: tbl0.Key(), - Shards: dax.NewShardNums(1, 3), - }, + Table: tbl0.Key(), + Shards: dax.NewShardNums(1, 3), }, }, }, @@ -896,13 +982,11 @@ func TestController(t *testing.T) { TableKey: tbl0.Key(), Shards: dax.NewShardNums(1), }, - exp: []dax.AssignedNode{ + exp: []dax.ComputeNode{ { Address: node1.Address, - Role: &dax.ComputeRole{ - TableKey: tbl0.Key(), - Shards: dax.NewShardNums(1), - }, + Table: tbl0.Key(), + Shards: dax.NewShardNums(1), }, }, }, @@ -913,20 +997,16 @@ func TestController(t *testing.T) { Shards: dax.NewShardNums(1, 888, 889), }, isWrite: true, - exp: []dax.AssignedNode{ + exp: []dax.ComputeNode{ { Address: node0.Address, - Role: &dax.ComputeRole{ - TableKey: tbl0.Key(), - Shards: dax.NewShardNums(888), - }, + Table: tbl0.Key(), + Shards: dax.NewShardNums(888), }, { Address: node1.Address, - Role: &dax.ComputeRole{ - TableKey: tbl0.Key(), - Shards: dax.NewShardNums(1, 889), - }, + Table: tbl0.Key(), + Shards: dax.NewShardNums(1, 889), }, }, }, @@ -936,31 +1016,32 @@ func TestController(t *testing.T) { TableKey: tbl0.Key(), Shards: dax.NewShardNums(2, 11), }, - exp: []dax.AssignedNode{ + exp: []dax.ComputeNode{ { Address: node0.Address, - Role: &dax.ComputeRole{ - TableKey: tbl0.Key(), - Shards: dax.NewShardNums(2, 11), - }, + Table: tbl0.Key(), + Shards: dax.NewShardNums(2, 11), }, }, }, } for i, test := range tests { t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { - nodes, err := con.Nodes(ctx, test.role, test.isWrite) + if test.isWrite { + addShards(t, ctx, con, test.role.TableKey.QualifiedTableID(), test.role.Shards...) + } + nodes, err := con.ComputeNodes(ctx, test.role.TableKey.QualifiedTableID(), test.role.Shards) assert.NoError(t, err) assert.Equal(t, test.exp, nodes) }) } }) - t.Run("TranslateRole", func(t *testing.T) { + t.Run("TranslateNodes", func(t *testing.T) { tests := []struct { - role dax.Role + role *dax.TranslateRole isWrite bool - exp []dax.AssignedNode + exp []dax.TranslateNode expErrCode errors.Code }{ { @@ -969,13 +1050,11 @@ func TestController(t *testing.T) { Partitions: dax.NewPartitionNums(0), }, isWrite: true, - exp: []dax.AssignedNode{ + exp: []dax.TranslateNode{ { - Address: node0.Address, - Role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.NewPartitionNums(0), - }, + Address: node0.Address, + Table: tbl0.Key(), + Partitions: dax.NewPartitionNums(0), }, }, }, @@ -985,22 +1064,19 @@ func TestController(t *testing.T) { Partitions: dax.NewPartitionNums(0, 1, 2, 3, 999), }, isWrite: false, - exp: []dax.AssignedNode{ + exp: []dax.TranslateNode{ { - Address: node0.Address, - Role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.NewPartitionNums(0, 2), - }, + Address: node0.Address, + Table: tbl0.Key(), + Partitions: dax.NewPartitionNums(0, 2), }, { - Address: node1.Address, - Role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.NewPartitionNums(1, 3), - }, + Address: node1.Address, + Table: tbl0.Key(), + Partitions: dax.NewPartitionNums(1, 3), }, }, + expErrCode: controller.ErrCodeUnassignedJobs, }, { role: &dax.TranslateRole{ @@ -1008,13 +1084,11 @@ func TestController(t *testing.T) { Partitions: dax.NewPartitionNums(1), }, isWrite: false, - exp: []dax.AssignedNode{ + exp: []dax.TranslateNode{ { - Address: node1.Address, - Role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.NewPartitionNums(1), - }, + Address: node1.Address, + Table: tbl0.Key(), + Partitions: dax.NewPartitionNums(1), }, }, }, @@ -1025,20 +1099,21 @@ func TestController(t *testing.T) { Partitions: dax.NewPartitionNums(2, 10), }, isWrite: false, - exp: []dax.AssignedNode{ + exp: []dax.TranslateNode{ { - Address: node0.Address, - Role: &dax.TranslateRole{ - TableKey: tbl0.Key(), - Partitions: dax.NewPartitionNums(2, 10), - }, + Address: node0.Address, + Table: tbl0.Key(), + Partitions: dax.NewPartitionNums(2, 10), }, }, }, } for i, test := range tests { t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { - nodes, err := con.Nodes(ctx, test.role, test.isWrite) + if test.isWrite { + addPartitions(t, ctx, con, test.role.TableKey.QualifiedTableID(), test.role.Partitions...) + } + nodes, err := con.TranslateNodes(ctx, test.role.TableKey.QualifiedTableID(), test.role.Partitions) if test.expErrCode != "" { assert.True(t, errors.Is(err, test.expErrCode)) @@ -1101,3 +1176,21 @@ func (d *testDirector) flush() []*dax.Directive { return out } + +func addShards(t *testing.T, ctx context.Context, con *controller.Controller, qtid dax.QualifiedTableID, shards ...dax.ShardNum) { + t.Helper() + for _, shard := range shards { + if _, err := con.IngestShard(ctx, qtid, shard); err != nil { + assert.NoError(t, err) + } + } +} + +func addPartitions(t *testing.T, ctx context.Context, con *controller.Controller, qtid dax.QualifiedTableID, partitions ...dax.PartitionNum) { + t.Helper() + for _, parition := range partitions { + if _, err := con.IngestPartition(ctx, qtid, parition); err != nil { + assert.NoError(t, err) + } + } +} diff --git a/dax/mds/controller/errors.go b/dax/mds/controller/errors.go index 51b13cd59..f19bad93d 100644 --- a/dax/mds/controller/errors.go +++ b/dax/mds/controller/errors.go @@ -96,6 +96,6 @@ func NewErrInvalidRequest(msg string) error { func NewErrUnassignedJobs(jobs []dax.Job) error { return errors.New( ErrCodeUnassignedJobs, - fmt.Sprintf("found %d unassigned jobs", len(jobs)), + fmt.Sprintf("found %d unassigned jobs: %+v", len(jobs), jobs), ) } diff --git a/dax/mds/controller/naive/balancer.go b/dax/mds/controller/naive/balancer.go deleted file mode 100644 index 8a8d2a996..000000000 --- a/dax/mds/controller/naive/balancer.go +++ /dev/null @@ -1,599 +0,0 @@ -// Package naive contains a naive implementation of the Balancer interface. -package naive - -import ( - "context" - "fmt" - "log" - "math" - "sort" - "strings" - "sync" - "time" - - "github.com/featurebasedb/featurebase/v3/dax" - "github.com/featurebasedb/featurebase/v3/dax/mds/controller" - "github.com/featurebasedb/featurebase/v3/errors" - "github.com/featurebasedb/featurebase/v3/logger" -) - -// Ensure type implements interface. -var _ controller.Balancer = (*Balancer)(nil) - -// Balancer is a naive implementation of the controller.Balancer interface. It -// helps manage the relationships between workers and jobs. The logic it uses to -// balance jobs across workers is very simple; it bases everything off the -// number of workers and number of jobs. It does not take anything else (such as -// job size, worker capabilities, etc) into consideration. -type Balancer struct { - mu sync.RWMutex - - // name is used in logging to help identify the balancer responsible for the - // log. - name string - - // current represents the current state of worker/job assigments. - current WorkerJobService - - // freeJobs is the set of jobs which have yet to be assigned to a worker. - // This could be because there are no available workers, or because a worker - // has been removed and the jobs for which it was responsible have yet to be - // reassigned. - freeJobs FreeJobService - - logger logger.Logger -} - -type WorkerJobService interface { - WorkersJobs(ctx context.Context, balancerName string) ([]dax.WorkerInfo, error) - - WorkerCount(ctx context.Context, balancerName string) (int, error) - ListWorkers(ctx context.Context, balancerName string) (dax.Workers, error) - WorkerExists(ctx context.Context, balancerName string, worker dax.Worker) (bool, error) - CreateWorker(ctx context.Context, balancerName string, worker dax.Worker) error - DeleteWorker(ctx context.Context, balancerName string, worker dax.Worker) error - - CreateJobs(ctx context.Context, balancerName string, worker dax.Worker, job ...dax.Job) error - DeleteJob(ctx context.Context, balancerName string, worker dax.Worker, job dax.Job) error - DeleteJobs(ctx context.Context, balancerName, prefix string) (InternalDiffs, error) - JobCounts(ctx context.Context, balancerName string, worker ...dax.Worker) (map[dax.Worker]int, error) - ListJobs(ctx context.Context, balancerName string, worker dax.Worker) (dax.Jobs, error) -} - -// TODO: I don't think all these method names need "Free" in them. -type FreeJobService interface { - CreateFreeJobs(ctx context.Context, balancerName string, job ...dax.Job) error - DeleteFreeJob(ctx context.Context, balancerName string, job dax.Job) error - ListFreeJobs(ctx context.Context, balancerName string) (dax.Jobs, error) - MergeFreeJobs(ctx context.Context, balancerName string, jobs dax.Jobs) error - DeleteFreeJobs(ctx context.Context, balancerName, prefix string) error -} - -// New returns a new instance of Balancer. -func New(name string, fjs FreeJobService, wjs WorkerJobService, logger logger.Logger) *Balancer { - return &Balancer{ - name: name, - current: wjs, - freeJobs: fjs, - logger: logger, - } -} - -// AddWorker adds a worker to the Balancer's worker pool. This may cause the -// Balancer to assign existing jobs that are currently in the free list to the -// worker. Also, the worker will immediately be available for assignments of new -// jobs. -func (b *Balancer) AddWorker(ctx context.Context, worker fmt.Stringer) ([]dax.WorkerDiff, error) { - b.logger.Debugf("%s: AddWorker(%s)", b.name, worker.String()) - b.mu.Lock() - defer b.mu.Unlock() - - diff, err := b.addWorker(ctx, dax.Worker(worker.String())) - if err != nil { - return nil, errors.Wrap(err, "adding worker") - } - - return diff.Output(), nil -} - -func (b *Balancer) addWorker(ctx context.Context, worker dax.Worker) (InternalDiffs, error) { - // If this worker already exists, don't do anything. - if exists, err := b.current.WorkerExists(ctx, b.name, worker); err != nil { - return nil, errors.Wrap(err, "checking if worker exists") - } else if exists { - return InternalDiffs{}, nil - } - - if err := b.current.CreateWorker(ctx, b.name, worker); err != nil { - return nil, errors.Wrap(err, "creating worker") - } - - // Process the freeJobs. - return b.processFreeJobs(ctx) -} - -// ReplaceWorker is meant to avoid the job re-assignment caused by performing a -// RemoveWorker followed by an AddWorker. In this case, it does both in one step -// so that it's more likely that the jobs will just get transferred directly -// over. NOT IMPLEMENTED YET. -// func (b *Balancer) ReplaceWorker(fromWorker string, toWorker string) []WorkerDiff { -// b.mu.Lock() -// defer b.mu.Unlock() - -// return []WorkerDiff{} -// } - -// RemoveWorker removes a worker from the worker pool and moves any of its -// currently assigned jobs to the free list. If the intention is to remove a -// worker and reassign its jobs to other workers, then RemoveWorker() should be -// followed by Balance(). -func (b *Balancer) RemoveWorker(ctx context.Context, worker fmt.Stringer) ([]dax.WorkerDiff, error) { - b.mu.Lock() - defer b.mu.Unlock() - - diff, err := b.removeWorker(ctx, dax.Worker(worker.String())) - if err != nil { - return nil, errors.Wrap(err, "removing worker") - } - - return diff.Output(), nil -} - -func (b *Balancer) removeWorker(ctx context.Context, worker dax.Worker) (InternalDiffs, error) { - // If this worker doesn't exist, don't do anything else. - if exists, err := b.current.WorkerExists(ctx, b.name, worker); err != nil { - return nil, errors.Wrap(err, "checking if worker exists") - } else if !exists { - return InternalDiffs{}, nil - } - - jobs, err := b.current.ListJobs(ctx, b.name, worker) - if err != nil { - return nil, errors.Wrap(err, "listing jobs") - } - - // Before removing the worker, mark its jobs as free. - if err := b.freeJobs.MergeFreeJobs(ctx, b.name, jobs); err != nil { - return nil, errors.Wrap(err, "merging free jobs") - } - - // Remove the worker. - if err := b.current.DeleteWorker(ctx, b.name, worker); err != nil { - return nil, errors.Wrap(err, "deleting worker") - } - - // Even though this may not be useful to the caller (for example, in the - // case where the worker has died and no longer exists), return the diffs - // which represent the removal of jobs from the worker. - diff := NewInternalDiffs() - for _, job := range jobs { - diff.Removed(worker, job) - } - - return diff, nil -} - -// AddJobs adds one or more jobs to an existing worker. If there are no existing -// workers, the jobs are placed into the free list and will be assigned to a -// worker once one becomes available. -func (b *Balancer) AddJobs(ctx context.Context, jobs ...fmt.Stringer) ([]dax.WorkerDiff, error) { - start := time.Now() - defer func() { - log.Printf("ELAPSED: Balancer.AddJob: %v", time.Since(start)) - }() - - jobsToAdd := make([]dax.Job, 0, len(jobs)) - for _, job := range jobs { - jobsToAdd = append(jobsToAdd, dax.Job(job.String())) - } - - if len(jobsToAdd) == 1 { - b.logger.Debugf("%s: AddJobs (%s)", b.name, jobsToAdd[0]) - } else { - b.logger.Debugf("%s: AddJobs (%d)", b.name, len(jobsToAdd)) - } - - b.mu.Lock() - defer b.mu.Unlock() - - diff, err := b.addJobs(ctx, jobsToAdd...) - if err != nil { - return nil, errors.Wrap(err, "adding job") - } - - return diff.Output(), nil -} - -func (b *Balancer) addJobs(ctx context.Context, jobs ...dax.Job) (InternalDiffs, error) { - if cnt, err := b.current.WorkerCount(ctx, b.name); err != nil { - return nil, errors.Wrap(err, "getting worker count") - } else if cnt == 0 { - if err := b.freeJobs.CreateFreeJobs(ctx, b.name, jobs...); err != nil { - return nil, errors.Wrap(err, "creating free job") - } - // TODO: we might want to inform the user that a job is in the free list - // because there are no workers. - return InternalDiffs{}, nil - } - - workerJobs, err := b.current.WorkersJobs(ctx, b.name) - if err != nil { - return nil, errors.Wrapf(err, "getting workers jobs: %s", b.name) - } - jset := dax.NewSet[dax.Job]() - for _, workerInfo := range workerJobs { - jset.Merge(dax.NewSet(workerInfo.Jobs...)) - } - - workerIDs := make(dax.Workers, 0, len(workerJobs)) - jobCounts := make(map[dax.Worker]int, 0) - for _, v := range workerJobs { - workerIDs = append(workerIDs, v.ID) - jobCounts[v.ID] = len(v.Jobs) - } - - diffs := NewInternalDiffs() - - jobsToCreate := make(map[dax.Worker][]dax.Job) - - for _, job := range jobs { - // Skip any job that already exists. - if jset.Contains(job) { - continue - } - - // Find the worker with the fewest number of jobs and assign it this job. - var lowCount int = math.MaxInt - var lowWorker dax.Worker - - // We loop over workerIDs here instead of jobCounts because jobCounts is - // a map and it can return results in an unexpected order, which is a - // problem for testing. - for _, worker := range workerIDs { - jobCount := jobCounts[worker] - if jobCount < lowCount { - lowCount = jobCount - lowWorker = worker - } - } - - jobsToCreate[lowWorker] = append(jobsToCreate[lowWorker], job) - jobCounts[lowWorker]++ - } - - for worker, jobs := range jobsToCreate { - if err := b.current.CreateJobs(ctx, b.name, worker, jobs...); err != nil { - return nil, errors.Wrap(err, "creating job") - } - for _, job := range jobs { - diffs.Added(worker, job) - } - } - - return diffs, nil -} - -// RemoveJob removes a job from the worker to which is was assigned. If the job -// is not currently assigned to a worker, but it is in the free list, then it -// will be removed from the free list. -func (b *Balancer) RemoveJob(ctx context.Context, job fmt.Stringer) ([]dax.WorkerDiff, error) { - b.mu.Lock() - defer b.mu.Unlock() - - diff, err := b.removeJob(ctx, dax.Job(job.String())) - if err != nil { - return nil, errors.Wrapf(err, "removing job: %s", job) - } - - return diff.Output(), nil -} - -func (b *Balancer) RemoveJobs(ctx context.Context, prefix string) ([]dax.WorkerDiff, error) { - b.mu.Lock() - defer b.mu.Unlock() - - idiffs, err := b.current.DeleteJobs(ctx, b.name, prefix) - if err != nil { - return nil, errors.Wrap(err, "deleting worker jobs") - } - if err := b.freeJobs.DeleteFreeJobs(ctx, b.name, prefix); err != nil { - return nil, errors.Wrap(err, "deleting free jobs") - } - return idiffs.Output(), nil -} - -func (b *Balancer) removeJob(ctx context.Context, job dax.Job) (InternalDiffs, error) { - if worker, ok, err := b.workerForJob(ctx, job); err != nil { - return nil, errors.Wrapf(err, "getting worker for job: %s", job) - } else if ok { - if err := b.current.DeleteJob(ctx, b.name, worker, job); err != nil { - return nil, errors.Wrapf(err, "deleting job: %s", job) - } - - diffs := NewInternalDiffs() - diffs.Removed(worker, job) - - return diffs, nil - } - - // Just in case the job is in the free list (and wasn't assigned to a - // worker), remove it; there's no need to provide a diff. There should never - // be a case where the same job is both in the free list and assigned to a - // worker. - if err := b.freeJobs.DeleteFreeJob(ctx, b.name, job); err != nil { - return nil, errors.Wrapf(err, "deleting free job: %s", job) - } - - return InternalDiffs{}, nil -} - -// Balance ensures that all jobs are being handled by a worker by assigning jobs -// in the free list to workers, and by moving job assignments around in order to -// balance the load on workers. -func (b *Balancer) Balance(ctx context.Context) ([]dax.WorkerDiff, error) { - b.mu.Lock() - defer b.mu.Unlock() - - // If there are no workers, we can't properly balance. - if cnt, err := b.current.WorkerCount(ctx, b.name); err != nil { - return nil, errors.Wrapf(err, "getting worker count: %s", b.name) - } else if cnt == 0 { - return []dax.WorkerDiff{}, nil - } - - // Process the freeJobs. - diffs, err := b.processFreeJobs(ctx) - if err != nil { - return nil, errors.Wrapf(err, "processing free jobs: %s", b.name) - } - - // Balance the jobs among workers. - diff, err := b.balance(ctx, diffs) - if err != nil { - return nil, errors.Wrap(err, "balancing jobs") - } - - return diff.Output(), nil -} - -// balance moves jobs among workers with the goal of having an equal number of -// jobs per worker. This method takes an `internalDiffs` as input for cases -// where some action has preceeded this call which also resulted in -// `internalDiffs`. Instead of having this method take a value, we could rely on -// the internalDiffs.merge() method, but we would need to modify that method to -// be smarter about the order in which it applies the add/remove operations. -// Until that's in place, we'll pass in a value here. -func (b *Balancer) balance(ctx context.Context, diffs InternalDiffs) (InternalDiffs, error) { - numWorkers, err := b.current.WorkerCount(ctx, b.name) - if err != nil { - return nil, errors.Wrapf(err, "getting worker count: %s", b.name) - } - numJobs := 0 - if workers, err := b.current.ListWorkers(ctx, b.name); err != nil { - return nil, errors.Wrapf(err, "listing workers: %s", b.name) - } else { - for _, worker := range workers { - jobCounts, err := b.current.JobCounts(ctx, b.name, worker) - if err != nil { - return nil, errors.Wrapf(err, "getting job count: %s", worker) - } - numJobs += jobCounts[worker] - } - } - - minJobsPerWorker := numJobs / numWorkers - numWorkersAboveMin := numJobs % numWorkers - - // sortedWorkerInfos is used now in order to guarantee a sort order. - sortedWorkerInfos, err := b.currentState(ctx, true) - if err != nil { - return nil, errors.Wrapf(err, "getting current state: %s", b.name) - } - - // Loop through each worker, and if the number of jobs for the worker - // exceeds the target, then remove the job and add it back (which is - // effectively how we rebalance a job). - for i, workerInfo := range sortedWorkerInfos { - numTargetJobs := minJobsPerWorker - if i < numWorkersAboveMin { - numTargetJobs += 1 - } - - jobCounts, err := b.current.JobCounts(ctx, b.name, workerInfo.ID) - if err != nil { - return nil, errors.Wrapf(err, "getting job count: %s", workerInfo.ID) - } - numCurrentJobs := jobCounts[workerInfo.ID] - - // If we don't need to remove jobs from this worker, then just continue - // on to the next worker. - if numCurrentJobs <= numTargetJobs { - continue - } - - sortedJobs, err := b.current.ListJobs(ctx, b.name, workerInfo.ID) - if err != nil { - return nil, errors.Wrapf(err, "listing jobs: %s", workerInfo.ID) - } - - // Remove the extra jobs from the end of the list, and add them back - // again (which should place them on a worker with fewer jobs). - for i := numCurrentJobs - 1; i >= numTargetJobs; i-- { - if rj, err := b.removeJob(ctx, sortedJobs[i]); err != nil { - return nil, errors.Wrapf(err, "removing job: %s", sortedJobs[i]) - } else { - diffs.Merge(rj) - } - if aj, err := b.addJobs(ctx, sortedJobs[i]); err != nil { - return nil, errors.Wrapf(err, "adding job: %s", sortedJobs[i]) - } else { - diffs.Merge(aj) - } - } - } - - return diffs, nil -} - -// CurrentState returns the current state of worker and job assignments. Note -// that there could be unassigned jobs which are not captured in this output. -// Calling Balance() would force any unassigned jobs to be assigned (assuming -// there is at least one worker), and the output would then reflect that. -func (b *Balancer) CurrentState(ctx context.Context) ([]dax.WorkerInfo, error) { - b.mu.RLock() - defer b.mu.RUnlock() - - return b.currentState(ctx, true) -} - -func (b *Balancer) currentState(ctx context.Context, sorted bool) ([]dax.WorkerInfo, error) { - return b.current.WorkersJobs(ctx, b.name) -} - -// WorkerState returns the current state of job assignments for a given worker. -func (b *Balancer) WorkerState(ctx context.Context, worker dax.Worker) (dax.WorkerInfo, error) { - b.mu.RLock() - defer b.mu.RUnlock() - - return b.workerState(ctx, worker) -} - -func (b *Balancer) workerState(ctx context.Context, worker dax.Worker) (dax.WorkerInfo, error) { - if exists, err := b.current.WorkerExists(ctx, b.name, worker); err != nil { - return dax.WorkerInfo{}, errors.Wrapf(err, "checking worker exists: %s", worker) - } else if !exists { - return dax.WorkerInfo{ - ID: dax.Worker(worker), - }, nil - } - - jobs, err := b.current.ListJobs(ctx, b.name, worker) - if err != nil { - return dax.WorkerInfo{}, errors.Wrapf(err, "listing jobs: %s", worker) - } - - return dax.WorkerInfo{ - ID: dax.Worker(worker), - Jobs: jobs, - }, nil -} - -// WorkersForJobs returns the list of workers for the given jobs. If a given job -// is not currently assigned to a worker, it will be ignored. -func (b *Balancer) WorkersForJobs(ctx context.Context, jobs []dax.Job) ([]dax.WorkerInfo, error) { - b.mu.RLock() - defer b.mu.RUnlock() - - return b.workersForJobs(ctx, jobs) -} - -func (b *Balancer) workersForJobs(ctx context.Context, jobs []dax.Job) ([]dax.WorkerInfo, error) { - out := make(map[dax.Worker]dax.Set[dax.Job]) - - workerJobs, err := b.current.WorkersJobs(ctx, b.name) - if err != nil { - return nil, errors.Wrapf(err, "getting worker jobs: %s", b.name) - } - for _, workerInfo := range workerJobs { - jset := dax.NewSet(workerInfo.Jobs...) - - matches := dax.NewSet[dax.Job]() - for _, job := range jobs { - if jset.Contains(job) { - matches.Add(job) - } - } - - if len(matches) > 0 { - out[workerInfo.ID] = matches - } - } - - workers := make([]dax.WorkerInfo, len(out)) - - i := 0 - for w, jset := range out { - workers[i] = dax.WorkerInfo{ - ID: dax.Worker(w), - Jobs: jset.Sorted(), - } - i++ - } - - sort.Sort(dax.WorkerInfos(workers)) - - return workers, nil -} - -func (b *Balancer) WorkersForJobPrefix(ctx context.Context, prefix string) ([]dax.WorkerInfo, error) { - b.mu.RLock() - defer b.mu.RUnlock() - - jobs, err := b.freeJobs.ListFreeJobs(ctx, b.name) - if err != nil { - return nil, errors.Wrap(err, "listing free jobs") - } - for _, job := range jobs { - if strings.HasPrefix(string(job), prefix) { - return nil, errors.Errorf("found free job '%s' matching prefix '%s'", job, prefix) - } - } - - workerJobs, err := b.current.WorkersJobs(ctx, b.name) - if err != nil { - return nil, errors.Wrapf(err, "getting worker jobs: %s", b.name) - } - - result := make([]dax.WorkerInfo, 0) - for _, workerInfo := range workerJobs { - matchedJobs := make([]dax.Job, 0) - for _, job := range workerInfo.Jobs { - if strings.HasPrefix(string(job), prefix) { - matchedJobs = append(matchedJobs, job) - } - } - if len(matchedJobs) > 0 { - result = append(result, dax.WorkerInfo{ - ID: workerInfo.ID, - Jobs: matchedJobs, - }) - } - } - return result, nil - -} - -// processFreeJobs assigns all jobs in the free list to a worker. -func (b *Balancer) processFreeJobs(ctx context.Context) (InternalDiffs, error) { - diffs := NewInternalDiffs() - jobs, err := b.freeJobs.ListFreeJobs(ctx, b.name) - if err != nil { - return nil, errors.Wrapf(err, "listing free jobs: %s", b.name) - } - for _, job := range jobs { - if aj, err := b.addJobs(ctx, job); err != nil { - return nil, errors.Wrapf(err, "adding job: %s", job) - } else { - diffs.Merge(aj) - } - if err := b.freeJobs.DeleteFreeJob(ctx, b.name, job); err != nil { - return nil, errors.Wrapf(err, "deleting free job: %s", job) - } - } - return diffs, nil -} - -// workerForJob returns the worker currently assigned to the given job. -func (b *Balancer) workerForJob(ctx context.Context, job dax.Job) (dax.Worker, bool, error) { - workerJobs, err := b.current.WorkersJobs(ctx, b.name) - if err != nil { - return "", false, errors.Wrapf(err, "getting workers jobs: %s", b.name) - } - for _, workerInfo := range workerJobs { - jset := dax.NewSet(workerInfo.Jobs...) - if jset.Contains(job) { - return workerInfo.ID, true, nil - } - } - return "", false, nil -} diff --git a/dax/mds/controller/naive/balancer_test.go b/dax/mds/controller/naive/balancer_test.go deleted file mode 100644 index 2830789e7..000000000 --- a/dax/mds/controller/naive/balancer_test.go +++ /dev/null @@ -1,751 +0,0 @@ -package naive_test - -import ( - "context" - "fmt" - "os" - "testing" - - "github.com/featurebasedb/featurebase/v3/dax" - daxbolt "github.com/featurebasedb/featurebase/v3/dax/boltdb" - "github.com/featurebasedb/featurebase/v3/dax/mds/controller/naive/boltdb" - testbolt "github.com/featurebasedb/featurebase/v3/dax/test/boltdb" - "github.com/featurebasedb/featurebase/v3/logger" - "github.com/stretchr/testify/assert" -) - -func newBoltBalancer(t *testing.T) (*daxbolt.DB, func()) { - db := testbolt.MustOpenDB(t) - assert.NoError(t, db.InitializeBuckets(boltdb.NaiveBalancerBuckets...)) - - return db, func() { - testbolt.MustCloseDB(t, db) - testbolt.CleanupDB(t, db.Path()) - } -} - -func TestBalancer(t *testing.T) { - ctx := context.Background() - t.Run("SingleWorker", func(t *testing.T) { - db, cleanup := newBoltBalancer(t) - defer cleanup() - bal := boltdb.NewBalancer("test", db, logger.NewStandardLogger(os.Stderr)) - // addJob is a wrapper around bal.AddJobs() which we added when the - // function signature of bal.AddJobs changed to take multiple jobs (and - // it therefore no longer satisfied the fn type in this test). - addJob := func(ctx context.Context, job fmt.Stringer) ([]dax.WorkerDiff, error) { - return bal.AddJobs(ctx, job) - } - tests := []struct { - fn func(context.Context, fmt.Stringer) ([]dax.WorkerDiff, error) - input string - expDiff []dax.WorkerDiff - expState []dax.WorkerInfo - }{ - { - // Add job. - fn: addJob, - input: "p2", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{}, - }, - { - // Add worker. - fn: bal.AddWorker, - input: "n1", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{"p2"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p2"}, - }, - }, - }, - { - // Add another job out of order. - fn: addJob, - input: "p1", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{"p1"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - }, - }, - { - // Add another job. - fn: addJob, - input: "p3", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{"p3"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2", "p3"}, - }, - }, - }, - { - // Add a duplicate job. - fn: addJob, - input: "p2", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2", "p3"}, - }, - }, - }, - } - for i, test := range tests { - t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { - diff, err := test.fn(ctx, newStringWrapper(test.input)) - assert.NoError(t, err) - assert.Equal(t, test.expDiff, diff) - - cs, err := bal.CurrentState(ctx) - assert.NoError(t, err) - assert.Equal(t, test.expState, cs) - }) - } - }) - - t.Run("MultipleWorkers", func(t *testing.T) { - db, cleanup := newBoltBalancer(t) - defer cleanup() - bal := boltdb.NewBalancer("test", db, logger.NewStandardLogger(os.Stderr)) - // addJob is a wrapper around bal.AddJobs() which we added when the - // function signature of bal.AddJobs changed to take multiple jobs (and - // it therefore no longer satisfied the fn type in this test). - addJob := func(ctx context.Context, job fmt.Stringer) ([]dax.WorkerDiff, error) { - return bal.AddJobs(ctx, job) - } - tests := []struct { - fn func(context.Context, fmt.Stringer) ([]dax.WorkerDiff, error) - input string - balance bool - expDiff []dax.WorkerDiff - expState []dax.WorkerInfo - }{ - { - // Balance when empty. - balance: true, - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{}, - }, - { - // Add worker. - fn: bal.AddWorker, - input: "n2", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n2", - Jobs: []dax.Job{}, - }, - }, - }, - { - // Add worker again. - fn: bal.AddWorker, - input: "n2", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n2", - Jobs: []dax.Job{}, - }, - }, - }, - { - // Add a second worker. - fn: bal.AddWorker, - input: "n1", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{}, - }, - { - ID: "n2", - Jobs: []dax.Job{}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p2", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{"p2"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p3", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n2", - AddedJobs: []dax.Job{"p3"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p3"}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p1", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{"p1"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p3"}, - }, - }, - }, - { - // Add a third worker. - fn: bal.AddWorker, - input: "n0", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p3"}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p4", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n0", - AddedJobs: []dax.Job{"p4"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4"}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p3"}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p5", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n0", - AddedJobs: []dax.Job{"p5"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5"}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p3"}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p0", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n2", - AddedJobs: []dax.Job{"p0"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5"}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p6", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n0", - AddedJobs: []dax.Job{"p6"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6"}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p7", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{"p7"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6"}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2", "p7"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - - //////////////////// Remove ///////////////////////// - - { - // Remove nonexistent worker. - fn: bal.RemoveWorker, - input: "nonexistent", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6"}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2", "p7"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - { - // Remove worker. - fn: bal.RemoveWorker, - input: "n1", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{}, - RemovedJobs: []dax.Job{"p1", "p2", "p7"}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - - { - // Remove job (from free list). - fn: bal.RemoveJob, - input: "p2", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - - { - // Balance after remove. - balance: true, - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n0", - AddedJobs: []dax.Job{"p7"}, - RemovedJobs: []dax.Job{}, - }, - { - WorkerID: "n2", - AddedJobs: []dax.Job{"p1"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6", "p7"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p1", "p3"}, - }, - }, - }, - - { - // Remove job. - fn: bal.RemoveJob, - input: "p1", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n2", - AddedJobs: []dax.Job{}, - RemovedJobs: []dax.Job{"p1"}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6", "p7"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - } - for i, test := range tests { - t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { - var diff []dax.WorkerDiff - var err error - if test.balance { - diff, err = bal.Balance(ctx) - } else { - diff, err = test.fn(ctx, newStringWrapper(test.input)) - } - assert.NoError(t, err) - assert.Equal(t, test.expDiff, diff) - - cs, err := bal.CurrentState(ctx) - assert.NoError(t, err) - assert.Equal(t, test.expState, cs) - }) - } - }) - - t.Run("WorkerState", func(t *testing.T) { - db, cleanup := newBoltBalancer(t) - defer cleanup() - bal := boltdb.NewBalancer("test", db, logger.NewStandardLogger(os.Stderr)) - - _, err := bal.AddWorker(ctx, newStringWrapper("n1")) - assert.NoError(t, err) - _, err = bal.AddJobs(ctx, newStringWrapper("p1")) - assert.NoError(t, err) - - exp := dax.WorkerInfo{ - ID: "n1", - Jobs: []dax.Job{"p1"}, - } - ws, err := bal.WorkerState(ctx, "n1") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - // Worker doesn't exist. - exp = dax.WorkerInfo{ - ID: "x1", - } - ws, err = bal.WorkerState(ctx, "x1") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - }) - - t.Run("WorkersForJobs", func(t *testing.T) { - db, cleanup := newBoltBalancer(t) - defer cleanup() - bal := boltdb.NewBalancer("test", db, logger.NewStandardLogger(os.Stderr)) - - _, err := bal.AddWorker(ctx, newStringWrapper("n1")) - assert.NoError(t, err) - _, err = bal.AddWorker(ctx, newStringWrapper("n2")) - assert.NoError(t, err) - for i := 0; i < 12; i++ { - _, err = bal.AddJobs(ctx, newStringWrapper(fmt.Sprintf("p%d", i))) - assert.NoError(t, err) - } - - exp := dax.WorkerInfo{ - ID: "n1", - Jobs: []dax.Job{"p0", "p10", "p2", "p4", "p6", "p8"}, - } - ws, err := bal.WorkerState(ctx, "n1") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - exp = dax.WorkerInfo{ - ID: "n2", - Jobs: []dax.Job{"p1", "p11", "p3", "p5", "p7", "p9"}, - } - ws, err = bal.WorkerState(ctx, "n2") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - tests := []struct { - jobs []dax.Job - exp []dax.WorkerInfo - }{ - { - jobs: []dax.Job{"p0"}, - exp: []dax.WorkerInfo{ - {ID: "n1", Jobs: []dax.Job{"p0"}}, - }, - }, - { - jobs: []dax.Job{"p0", "p4"}, - exp: []dax.WorkerInfo{ - {ID: "n1", Jobs: []dax.Job{"p0", "p4"}}, - }, - }, - { - jobs: []dax.Job{"p0", "p4", "p999"}, - exp: []dax.WorkerInfo{ - {ID: "n1", Jobs: []dax.Job{"p0", "p4"}}, - }, - }, - { - jobs: []dax.Job{"p0", "p1"}, - exp: []dax.WorkerInfo{ - {ID: "n1", Jobs: []dax.Job{"p0"}}, - {ID: "n2", Jobs: []dax.Job{"p1"}}, - }, - }, - { - jobs: []dax.Job{"p5", "p0", "p1", "p8"}, - exp: []dax.WorkerInfo{ - {ID: "n1", Jobs: []dax.Job{"p0", "p8"}}, - {ID: "n2", Jobs: []dax.Job{"p1", "p5"}}, - }, - }, - } - for i, test := range tests { - t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { - workers, err := bal.WorkersForJobs(ctx, test.jobs) - assert.NoError(t, err) - assert.Equal(t, test.exp, workers) - }) - } - - // Some tests for WorkersForJobPrefix - workers, err := bal.WorkersForJobPrefix(ctx, "p1") - assert.NoError(t, err) - assert.ElementsMatch(t, []dax.WorkerInfo{ - {ID: "n1", Jobs: []dax.Job{"p10"}}, - {ID: "n2", Jobs: []dax.Job{"p1", "p11"}}, - }, workers) - - workers, err = bal.WorkersForJobPrefix(ctx, "p2") - assert.NoError(t, err) - assert.ElementsMatch(t, []dax.WorkerInfo{ - {ID: "n1", Jobs: []dax.Job{"p2"}}, - }, workers) - - workers, err = bal.WorkersForJobPrefix(ctx, "pp") - assert.NoError(t, err) - assert.ElementsMatch(t, []dax.WorkerInfo{}, workers) - - }) - - t.Run("Balance", func(t *testing.T) { - db, cleanup := newBoltBalancer(t) - defer cleanup() - bal := boltdb.NewBalancer("test", db, logger.NewStandardLogger(os.Stderr)) - - // Add two workers with some jobs evenly spread across them. - _, err := bal.AddWorker(ctx, newStringWrapper("n1")) - assert.NoError(t, err) - _, err = bal.AddWorker(ctx, newStringWrapper("n2")) - assert.NoError(t, err) - for i := 0; i < 13; i++ { - _, err = bal.AddJobs(ctx, newStringWrapper(fmt.Sprintf("p%d", i))) - assert.NoError(t, err) - } - - exp := dax.WorkerInfo{ - ID: "n1", - Jobs: []dax.Job{"p0", "p10", "p12", "p2", "p4", "p6", "p8"}, - } - ws, err := bal.WorkerState(ctx, "n1") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - exp = dax.WorkerInfo{ - ID: "n2", - Jobs: []dax.Job{"p1", "p11", "p3", "p5", "p7", "p9"}, - } - ws, err = bal.WorkerState(ctx, "n2") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - // Now, add a worker and confirm that it currently has no jobs assigned - // to it. - _, err = bal.AddWorker(ctx, newStringWrapper("n3")) - assert.NoError(t, err) - exp = dax.WorkerInfo{ - ID: "n3", - Jobs: []dax.Job{}, - } - ws, err = bal.WorkerState(ctx, "n3") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - // Finally, call Balance() and confirm that the appropriate jobs got - // reassigned. - _, err = bal.Balance(ctx) - assert.NoError(t, err) - - exp = dax.WorkerInfo{ - ID: "n1", - Jobs: []dax.Job{"p0", "p10", "p12", "p2", "p4"}, - } - ws, err = bal.WorkerState(ctx, "n1") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - exp = dax.WorkerInfo{ - ID: "n2", - Jobs: []dax.Job{"p1", "p11", "p3", "p5"}, - } - ws, err = bal.WorkerState(ctx, "n2") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - exp = dax.WorkerInfo{ - ID: "n3", - Jobs: []dax.Job{"p6", "p7", "p8", "p9"}, - } - ws, err = bal.WorkerState(ctx, "n3") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - }) -} - -type stringWrapper struct { - s string -} - -func newStringWrapper(s string) *stringWrapper { - return &stringWrapper{ - s: s, - } -} - -func (s *stringWrapper) String() string { - return s.s -} diff --git a/dax/mds/controller/naive/boltdb/balancer.go b/dax/mds/controller/naive/boltdb/balancer.go deleted file mode 100644 index b8e202913..000000000 --- a/dax/mds/controller/naive/boltdb/balancer.go +++ /dev/null @@ -1,606 +0,0 @@ -// Package boltdb contains the boltdb implementation of the Balancer interface. -package boltdb - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "strings" - - "github.com/featurebasedb/featurebase/v3/dax" - "github.com/featurebasedb/featurebase/v3/dax/boltdb" - "github.com/featurebasedb/featurebase/v3/dax/mds/controller" - "github.com/featurebasedb/featurebase/v3/dax/mds/controller/naive" - "github.com/featurebasedb/featurebase/v3/errors" - "github.com/featurebasedb/featurebase/v3/logger" -) - -var ( - bucketNaiveBalancer = boltdb.Bucket("naiveBalancer") -) - -// NaiveBalancerBuckets defines the buckets used by this package. It can be -// called during setup to create the buckets ahead of time. -var NaiveBalancerBuckets []boltdb.Bucket = []boltdb.Bucket{ - bucketNaiveBalancer, -} - -// NewBalancer returns a new instance of controller.Balancer. -func NewBalancer(name string, db *boltdb.DB, logger logger.Logger) controller.Balancer { - fjs := newFreeJobService(db) - wjs := newWorkerJobService(db, logger) - - return naive.New(name, fjs, wjs, logger) -} - -// Ensure type implements interface. -var _ naive.WorkerJobService = (*workerJobService)(nil) - -type workerJobService struct { - db *boltdb.DB - logger logger.Logger -} - -func newWorkerJobService(db *boltdb.DB, logger logger.Logger) *workerJobService { - return &workerJobService{ - db: db, - logger: logger, - } -} - -func (w *workerJobService) WorkersJobs(ctx context.Context, balancerName string) ([]dax.WorkerInfo, error) { - tx, err := w.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrap(err, "getting tx") - } - defer tx.Rollback() - - workerInfos, err := getWorkerInfos(ctx, tx, balancerName) - if err != nil { - return nil, errors.Wrapf(err, "getting worker infos: %s", balancerName) - } - - return workerInfos, nil -} - -func (w *workerJobService) WorkerCount(ctx context.Context, balancerName string) (int, error) { - tx, err := w.db.BeginTx(ctx, false) - if err != nil { - return 0, errors.Wrap(err, "getting tx") - } - defer tx.Rollback() - - workers, err := w.getWorkers(ctx, tx, balancerName) - if err != nil { - return 0, errors.Wrapf(err, "getting workers: %s", balancerName) - } - - return len(workers), nil -} - -func (w *workerJobService) ListWorkers(ctx context.Context, balancerName string) (dax.Workers, error) { - tx, err := w.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - workers, err := w.getWorkers(ctx, tx, balancerName) - if err != nil { - return nil, errors.Wrapf(err, "getting workers: %s", balancerName) - } - - return workers, nil -} - -func (w *workerJobService) getWorkers(ctx context.Context, tx *boltdb.Tx, balancerName string) (dax.Workers, error) { - c := tx.Bucket(bucketNaiveBalancer).Cursor() - - // Deserialize rows into Worker objects. - workers := make(dax.Workers, 0) - - prefix := []byte(fmt.Sprintf(prefixFmtWorkers, balancerName)) - for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { - if v == nil { - w.logger.Printf("nil value for key: %s", k) - continue - } - - worker, err := keyWorker(k) - if err != nil { - return nil, errors.Wrapf(err, "getting worker from key: %s", k) - } - - workers = append(workers, worker) - } - - return workers, nil -} - -func getWorkerInfos(ctx context.Context, tx *boltdb.Tx, balancerName string) (dax.WorkerInfos, error) { - c := tx.Bucket(bucketNaiveBalancer).Cursor() - - // Deserialize rows into WorkerInfo objects. - workerInfos := make(dax.WorkerInfos, 0) - - prefix := []byte(fmt.Sprintf(prefixFmtWorkers, balancerName)) - for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { - worker, err := keyWorker(k) - if err != nil { - return nil, errors.Wrapf(err, "getting worker from key: %s", k) - } - - jobs := dax.NewSet[dax.Job]() - if v != nil { - jobs, err = decodeJobSet(v) - if err != nil { - return nil, errors.Wrap(err, "decoding job set") - } - } - - workerInfo := dax.WorkerInfo{ - ID: worker, - Jobs: jobs.Sorted(), - } - - workerInfos = append(workerInfos, workerInfo) - } - - return workerInfos, nil -} - -func (w *workerJobService) WorkerExists(ctx context.Context, balancerName string, worker dax.Worker) (bool, error) { - tx, err := w.db.BeginTx(ctx, false) - if err != nil { - return false, errors.Wrapf(err, "getting tx: %s", balancerName) - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketNaiveBalancer) - if bkt == nil { - return false, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) - } - - wrkr := bkt.Get(workerKey(balancerName, worker)) - - return wrkr != nil, nil -} - -func (w *workerJobService) CreateWorker(ctx context.Context, balancerName string, worker dax.Worker) error { - tx, err := w.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "getting transaction") - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketNaiveBalancer) - if bkt == nil { - return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) - } - - // If this worker already exists, don't do anything. - wrkr := bkt.Get(workerKey(balancerName, worker)) - if wrkr != nil { - return nil - } - - val := []byte("[]") - if err := bkt.Put(workerKey(balancerName, worker), val); err != nil { - return errors.Wrap(err, "putting worker") - } - - return tx.Commit() -} - -func (w *workerJobService) DeleteWorker(ctx context.Context, balancerName string, worker dax.Worker) error { - tx, err := w.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketNaiveBalancer) - if bkt == nil { - return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) - } - - if err := bkt.Delete(workerKey(balancerName, worker)); err != nil { - return errors.Wrapf(err, "deleting node key: %s", workerKey(balancerName, worker)) - } - - return tx.Commit() -} - -func (w *workerJobService) CreateJobs(ctx context.Context, balancerName string, worker dax.Worker, jobs ...dax.Job) error { - tx, err := w.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketNaiveBalancer) - if bkt == nil { - return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) - } - - jobset := dax.NewSet[dax.Job]() - - // get worker - wrkr := bkt.Get(workerKey(balancerName, worker)) - if wrkr != nil { - jobset, err = decodeJobSet(wrkr) - if err != nil { - return errors.Wrap(err, "decoding job set") - } - } - - for _, job := range jobs { - jobset.Add(job) - } - val, err := encodeJobSet(jobset) - if err != nil { - return errors.Wrap(err, "encoding job set") - } - - if err := bkt.Put(workerKey(balancerName, worker), val); err != nil { - return errors.Wrap(err, "putting worker") - } - - return tx.Commit() -} - -func (w *workerJobService) DeleteJob(ctx context.Context, balancerName string, worker dax.Worker, job dax.Job) error { - tx, err := w.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketNaiveBalancer) - if bkt == nil { - return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) - } - - // get worker - wrkr := bkt.Get(workerKey(balancerName, worker)) - if wrkr == nil { - return nil - } - - jobset, err := decodeJobSet(wrkr) - if err != nil { - return errors.Wrap(err, "decoding job set") - } - if !jobset.Contains(job) { - return nil - } - - jobset.Remove(job) - val, err := encodeJobSet(jobset) - if err != nil { - return errors.Wrap(err, "encoding job set") - } - - if err := bkt.Put(workerKey(balancerName, worker), val); err != nil { - return errors.Wrap(err, "putting worker") - } - - return tx.Commit() -} - -func (w *workerJobService) DeleteJobs(ctx context.Context, balancerName, prefix string) (naive.InternalDiffs, error) { - tx, err := w.db.BeginTx(ctx, true) - if err != nil { - return nil, errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketNaiveBalancer) - if bkt == nil { - return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) - } - - workers, err := w.getWorkers(ctx, tx, balancerName) - if err != nil { - return nil, errors.Wrap(err, "getting workers") - } - - idiffs := naive.NewInternalDiffs() - for _, worker := range workers { - // get worker - wrkr := bkt.Get(workerKey(balancerName, worker)) - if wrkr == nil { - panic("didn't find worker that should... definitely exist") - } - jobset, err := decodeJobSet(wrkr) - if err != nil { - return nil, errors.Wrap(err, "decoding job set") - } - - jobs := jobset.RemovePrefix(prefix) - for _, job := range jobs { - idiffs.Removed(worker, job) - } - val, err := encodeJobSet(jobset) - if err != nil { - return nil, errors.Wrap(err, "encoding job set") - } - - if err := bkt.Put(workerKey(balancerName, worker), val); err != nil { - return nil, errors.Wrap(err, "putting worker") - } - - } - - return idiffs, tx.Commit() -} - -func (w *workerJobService) ListJobs(ctx context.Context, balancerName string, worker dax.Worker) (dax.Jobs, error) { - tx, err := w.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketNaiveBalancer) - if bkt == nil { - return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) - } - - jobset := dax.NewSet[dax.Job]() - - // get worker - wrkr := bkt.Get(workerKey(balancerName, worker)) - if wrkr != nil { - jobset, err = decodeJobSet(wrkr) - if err != nil { - return nil, errors.Wrap(err, "decoding job set") - } - } - - return jobset.Sorted(), nil -} - -func (w *workerJobService) JobCounts(ctx context.Context, balancerName string, workers ...dax.Worker) (map[dax.Worker]int, error) { - tx, err := w.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrapf(err, "getting tx: %s", balancerName) - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketNaiveBalancer) - if bkt == nil { - return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) - } - - m := make(map[dax.Worker]int) - - for _, worker := range workers { - jobset := dax.NewSet[dax.Job]() - - // get worker - wrkr := bkt.Get(workerKey(balancerName, worker)) - if wrkr != nil { - jobset, err = decodeJobSet(wrkr) - if err != nil { - return nil, errors.Wrap(err, "decoding job set") - } - } - - m[worker] = len(jobset) - } - - return m, nil -} - -// encodeJobSet encode the jobSet into a JSON array of strings. -func encodeJobSet(jobSet dax.Set[dax.Job]) ([]byte, error) { - arr := jobSet.Sorted() - b, err := json.Marshal(arr) - if err != nil { - return nil, errors.Wrap(err, "marshalling json") - } - return b, nil -} - -// decodeJobSet decode the string (a JSON array of strings) into jobSet. -func decodeJobSet(v []byte) (dax.Set[dax.Job], error) { - var arr []string - err := json.Unmarshal(v, &arr) - if err != nil { - return nil, errors.Wrap(err, "unmarshalling json") - } - - js := dax.NewSet[dax.Job]() - for _, s := range arr { - js.Add(dax.Job(s)) - } - - return js, nil -} - -// Ensure type implements interface. -var _ naive.FreeJobService = (*freeJobService)(nil) - -type freeJobService struct { - db *boltdb.DB -} - -func newFreeJobService(db *boltdb.DB) *freeJobService { - return &freeJobService{ - db: db, - } -} - -func (f *freeJobService) CreateFreeJobs(ctx context.Context, balancerName string, jobs ...dax.Job) error { - return f.MergeFreeJobs(ctx, balancerName, jobs) -} - -func (f *freeJobService) DeleteFreeJob(ctx context.Context, balancerName string, job dax.Job) error { - tx, err := f.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketNaiveBalancer) - if bkt == nil { - return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) - } - - // get free jobs - fjs := bkt.Get(freeJobKey(balancerName)) - if fjs == nil { - return nil - } - - jobset, err := decodeJobSet(fjs) - if err != nil { - return errors.Wrap(err, "decoding job set") - } - if !jobset.Contains(job) { - return nil - } - - jobset.Remove(job) - val, err := encodeJobSet(jobset) - if err != nil { - return errors.Wrap(err, "encoding job set") - } - - if err := bkt.Put(freeJobKey(balancerName), val); err != nil { - return errors.Wrap(err, "putting free job") - } - - return tx.Commit() -} - -func (f *freeJobService) DeleteFreeJobs(ctx context.Context, balancerName, prefix string) error { - tx, err := f.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketNaiveBalancer) - if bkt == nil { - return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) - } - - // get free jobs - fjs := bkt.Get(freeJobKey(balancerName)) - if fjs == nil { - return nil - } - - jobset, err := decodeJobSet(fjs) - if err != nil { - return errors.Wrap(err, "decoding job set") - } - - jobset.RemovePrefix(prefix) - val, err := encodeJobSet(jobset) - if err != nil { - return errors.Wrap(err, "encoding job set") - } - - if err := bkt.Put(freeJobKey(balancerName), val); err != nil { - return errors.Wrap(err, "putting free job") - } - - return tx.Commit() -} - -func (f *freeJobService) ListFreeJobs(ctx context.Context, balancerName string) (dax.Jobs, error) { - tx, err := f.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrap(err, "beginning tx") - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketNaiveBalancer) - if bkt == nil { - return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) - } - - jobset := dax.NewSet[dax.Job]() - - // get free jobs - fjs := bkt.Get(freeJobKey(balancerName)) - if fjs != nil { - jobset, err = decodeJobSet(fjs) - if err != nil { - return nil, errors.Wrap(err, "decoding job set") - } - } - - return jobset.Sorted(), nil -} - -func (f *freeJobService) MergeFreeJobs(ctx context.Context, balancerName string, jobs dax.Jobs) error { - tx, err := f.db.BeginTx(ctx, true) - if err != nil { - return err - } - defer tx.Rollback() - - bkt := tx.Bucket(bucketNaiveBalancer) - if bkt == nil { - return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketNaiveBalancer) - } - - jobset := dax.NewSet[dax.Job]() - - // get free jobs - fjs := bkt.Get(freeJobKey(balancerName)) - if fjs != nil { - jobset, err = decodeJobSet(fjs) - if err != nil { - return errors.Wrap(err, "decoding job set") - } - } - - for _, j := range jobs { - jobset.Add(j) - } - val, err := encodeJobSet(jobset) - if err != nil { - return errors.Wrap(err, "encoding job set") - } - - if err := bkt.Put(freeJobKey(balancerName), val); err != nil { - return errors.Wrap(err, "putting free job") - } - - return tx.Commit() -} - -////////////////////////////////////////////////////// - -const ( - prefixFmtWorkers = "workers/%s/" // %s - balancerName - prefixFmtFreeJobs = "freejobs/%s" // %s - balancerName -) - -// workerKey returns a key based on worker. -func workerKey(bal string, worker dax.Worker) []byte { - key := fmt.Sprintf(prefixFmtWorkers+"%s", bal, worker) - return []byte(key) -} - -// keyWorker gets the worker out of the key. -func keyWorker(key []byte) (dax.Worker, error) { - parts := strings.SplitN(string(key), "/", 3) - if len(parts) != 3 { - return "", errors.New(errors.ErrUncoded, "worker key format expected: `workers/balancer/worker`") - } - - return dax.Worker(parts[2]), nil -} - -// freeJobKey returns a key for all freeJobs. -func freeJobKey(bal string) []byte { - key := fmt.Sprintf(prefixFmtFreeJobs, bal) - return []byte(key) -} diff --git a/dax/mds/controller/naive/boltdb/balancer_test.go b/dax/mds/controller/naive/boltdb/balancer_test.go deleted file mode 100644 index b305f43c4..000000000 --- a/dax/mds/controller/naive/boltdb/balancer_test.go +++ /dev/null @@ -1,721 +0,0 @@ -package boltdb_test - -import ( - "context" - "fmt" - "testing" - - "github.com/featurebasedb/featurebase/v3/dax" - "github.com/featurebasedb/featurebase/v3/dax/mds/controller/naive/boltdb" - testbolt "github.com/featurebasedb/featurebase/v3/dax/test/boltdb" - "github.com/featurebasedb/featurebase/v3/logger" - "github.com/stretchr/testify/assert" -) - -func TestBalancer(t *testing.T) { - db := testbolt.MustOpenDB(t) - defer testbolt.MustCloseDB(t, db) - - t.Cleanup(func() { - testbolt.CleanupDB(t, db.Path()) - }) - - ctx := context.Background() - - // Initialize the buckets. - assert.NoError(t, db.InitializeBuckets(boltdb.NaiveBalancerBuckets...)) - - t.Run("SingleWorker", func(t *testing.T) { - bal := boltdb.NewBalancer("test-single-worker", db, logger.NopLogger) - // addJob is a wrapper around bal.AddJobs() which we added when the - // function signature of bal.AddJobs changed to take multiple jobs (and - // it therefore no longer satisfied the fn type in this test). - addJob := func(ctx context.Context, job fmt.Stringer) ([]dax.WorkerDiff, error) { - return bal.AddJobs(ctx, job) - } - tests := []struct { - fn func(context.Context, fmt.Stringer) ([]dax.WorkerDiff, error) - input string - expDiff []dax.WorkerDiff - expState []dax.WorkerInfo - }{ - { - // Add job. - fn: addJob, - input: "p2", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{}, - }, - { - // Add worker. - fn: bal.AddWorker, - input: "n1", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{"p2"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p2"}, - }, - }, - }, - { - // Add another job out of order. - fn: addJob, - input: "p1", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{"p1"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - }, - }, - { - // Add another job. - fn: addJob, - input: "p3", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{"p3"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2", "p3"}, - }, - }, - }, - { - // Add a duplicate job. - fn: addJob, - input: "p2", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2", "p3"}, - }, - }, - }, - } - for i, test := range tests { - t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { - diff, err := test.fn(ctx, newStringWrapper(test.input)) - assert.NoError(t, err) - assert.Equal(t, test.expDiff, diff) - - cs, err := bal.CurrentState(ctx) - assert.NoError(t, err) - assert.Equal(t, test.expState, cs) - }) - } - }) - - t.Run("MultipleWorkers", func(t *testing.T) { - bal := boltdb.NewBalancer("test-multiple-workers", db, logger.NopLogger) - // addJob is a wrapper around bal.AddJobs() which we added when the - // function signature of bal.AddJobs changed to take multiple jobs (and - // it therefore no longer satisfied the fn type in this test). - addJob := func(ctx context.Context, job fmt.Stringer) ([]dax.WorkerDiff, error) { - return bal.AddJobs(ctx, job) - } - tests := []struct { - fn func(context.Context, fmt.Stringer) ([]dax.WorkerDiff, error) - input string - balance bool - expDiff []dax.WorkerDiff - expState []dax.WorkerInfo - }{ - { - // Balance when empty. - balance: true, - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{}, - }, - { - // Add worker. - fn: bal.AddWorker, - input: "n2", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n2", - Jobs: []dax.Job{}, - }, - }, - }, - { - // Add worker again. - fn: bal.AddWorker, - input: "n2", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n2", - Jobs: []dax.Job{}, - }, - }, - }, - { - // Add a second worker. - fn: bal.AddWorker, - input: "n1", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{}, - }, - { - ID: "n2", - Jobs: []dax.Job{}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p2", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{"p2"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p3", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n2", - AddedJobs: []dax.Job{"p3"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p3"}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p1", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{"p1"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p3"}, - }, - }, - }, - { - // Add a third worker. - fn: bal.AddWorker, - input: "n0", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p3"}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p4", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n0", - AddedJobs: []dax.Job{"p4"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4"}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p3"}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p5", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n0", - AddedJobs: []dax.Job{"p5"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5"}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p3"}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p0", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n2", - AddedJobs: []dax.Job{"p0"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5"}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p6", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n0", - AddedJobs: []dax.Job{"p6"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6"}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - { - // Add job. - fn: addJob, - input: "p7", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{"p7"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6"}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2", "p7"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - - //////////////////// Remove ///////////////////////// - - { - // Remove nonexistent worker. - fn: bal.RemoveWorker, - input: "nonexistent", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6"}, - }, - { - ID: "n1", - Jobs: []dax.Job{"p1", "p2", "p7"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - { - // Remove worker. - fn: bal.RemoveWorker, - input: "n1", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n1", - AddedJobs: []dax.Job{}, - RemovedJobs: []dax.Job{"p1", "p2", "p7"}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - - { - // Remove job (from free list). - fn: bal.RemoveJob, - input: "p2", - expDiff: []dax.WorkerDiff{}, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - - { - // Balance after remove. - balance: true, - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n0", - AddedJobs: []dax.Job{"p7"}, - RemovedJobs: []dax.Job{}, - }, - { - WorkerID: "n2", - AddedJobs: []dax.Job{"p1"}, - RemovedJobs: []dax.Job{}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6", "p7"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p1", "p3"}, - }, - }, - }, - - { - // Remove job. - fn: bal.RemoveJob, - input: "p1", - expDiff: []dax.WorkerDiff{ - { - WorkerID: "n2", - AddedJobs: []dax.Job{}, - RemovedJobs: []dax.Job{"p1"}, - }, - }, - expState: []dax.WorkerInfo{ - { - ID: "n0", - Jobs: []dax.Job{"p4", "p5", "p6", "p7"}, - }, - { - ID: "n2", - Jobs: []dax.Job{"p0", "p3"}, - }, - }, - }, - } - for i, test := range tests { - t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { - var diff []dax.WorkerDiff - var err error - if test.balance { - diff, err = bal.Balance(ctx) - } else { - diff, err = test.fn(ctx, newStringWrapper(test.input)) - } - assert.NoError(t, err) - assert.Equal(t, test.expDiff, diff) - - cs, err := bal.CurrentState(ctx) - assert.NoError(t, err) - assert.Equal(t, test.expState, cs) - }) - } - }) - - t.Run("WorkerState", func(t *testing.T) { - bal := boltdb.NewBalancer("test-worker-state", db, logger.NopLogger) - - _, err := bal.AddWorker(ctx, newStringWrapper("n1")) - assert.NoError(t, err) - _, err = bal.AddJobs(ctx, newStringWrapper("p1")) - assert.NoError(t, err) - - exp := dax.WorkerInfo{ - ID: "n1", - Jobs: []dax.Job{"p1"}, - } - ws, err := bal.WorkerState(ctx, "n1") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - // Worker doesn't exist. - exp = dax.WorkerInfo{ - ID: "x1", - } - ws, err = bal.WorkerState(ctx, "x1") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - }) - - t.Run("WorkersForJobs", func(t *testing.T) { - bal := boltdb.NewBalancer("test-workers-for-jobs", db, logger.NopLogger) - - _, err := bal.AddWorker(ctx, newStringWrapper("n1")) - assert.NoError(t, err) - _, err = bal.AddWorker(ctx, newStringWrapper("n2")) - assert.NoError(t, err) - for i := 0; i < 12; i++ { - _, err = bal.AddJobs(ctx, newStringWrapper(fmt.Sprintf("p%d", i))) - assert.NoError(t, err) - } - - exp := dax.WorkerInfo{ - ID: "n1", - Jobs: []dax.Job{"p0", "p10", "p2", "p4", "p6", "p8"}, - } - ws, err := bal.WorkerState(ctx, "n1") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - exp = dax.WorkerInfo{ - ID: "n2", - Jobs: []dax.Job{"p1", "p11", "p3", "p5", "p7", "p9"}, - } - ws, err = bal.WorkerState(ctx, "n2") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - tests := []struct { - jobs []dax.Job - exp []dax.WorkerInfo - }{ - { - jobs: []dax.Job{"p0"}, - exp: []dax.WorkerInfo{ - {ID: "n1", Jobs: []dax.Job{"p0"}}, - }, - }, - { - jobs: []dax.Job{"p0", "p4"}, - exp: []dax.WorkerInfo{ - {ID: "n1", Jobs: []dax.Job{"p0", "p4"}}, - }, - }, - { - jobs: []dax.Job{"p0", "p4", "p999"}, - exp: []dax.WorkerInfo{ - {ID: "n1", Jobs: []dax.Job{"p0", "p4"}}, - }, - }, - { - jobs: []dax.Job{"p0", "p1"}, - exp: []dax.WorkerInfo{ - {ID: "n1", Jobs: []dax.Job{"p0"}}, - {ID: "n2", Jobs: []dax.Job{"p1"}}, - }, - }, - { - jobs: []dax.Job{"p5", "p0", "p1", "p8"}, - exp: []dax.WorkerInfo{ - {ID: "n1", Jobs: []dax.Job{"p0", "p8"}}, - {ID: "n2", Jobs: []dax.Job{"p1", "p5"}}, - }, - }, - } - for i, test := range tests { - t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { - workers, err := bal.WorkersForJobs(ctx, test.jobs) - assert.NoError(t, err) - assert.Equal(t, test.exp, workers) - }) - } - }) - - t.Run("Balance", func(t *testing.T) { - bal := boltdb.NewBalancer("test-balance", db, logger.NopLogger) - - // Add two workers with some jobs evenly spread across them. - _, err := bal.AddWorker(ctx, newStringWrapper("n1")) - assert.NoError(t, err) - _, err = bal.AddWorker(ctx, newStringWrapper("n2")) - assert.NoError(t, err) - for i := 0; i < 13; i++ { - _, err = bal.AddJobs(ctx, newStringWrapper(fmt.Sprintf("p%d", i))) - assert.NoError(t, err) - } - - exp := dax.WorkerInfo{ - ID: "n1", - Jobs: []dax.Job{"p0", "p10", "p12", "p2", "p4", "p6", "p8"}, - } - ws, err := bal.WorkerState(ctx, "n1") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - exp = dax.WorkerInfo{ - ID: "n2", - Jobs: []dax.Job{"p1", "p11", "p3", "p5", "p7", "p9"}, - } - ws, err = bal.WorkerState(ctx, "n2") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - // Now, add a worker and confirm that it currently has no jobs assigned - // to it. - _, err = bal.AddWorker(ctx, newStringWrapper("n3")) - assert.NoError(t, err) - exp = dax.WorkerInfo{ - ID: "n3", - Jobs: []dax.Job{}, - } - ws, err = bal.WorkerState(ctx, "n3") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - // Finally, call Balance() and confirm that the appropriate jobs got - // reassigned. - _, err = bal.Balance(ctx) - assert.NoError(t, err) - - exp = dax.WorkerInfo{ - ID: "n1", - Jobs: []dax.Job{"p0", "p10", "p12", "p2", "p4"}, - } - ws, err = bal.WorkerState(ctx, "n1") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - exp = dax.WorkerInfo{ - ID: "n2", - Jobs: []dax.Job{"p1", "p11", "p3", "p5"}, - } - ws, err = bal.WorkerState(ctx, "n2") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - - exp = dax.WorkerInfo{ - ID: "n3", - Jobs: []dax.Job{"p6", "p7", "p8", "p9"}, - } - ws, err = bal.WorkerState(ctx, "n3") - assert.NoError(t, err) - assert.Equal(t, exp, ws) - }) -} - -type stringWrapper struct { - s string -} - -func newStringWrapper(s string) *stringWrapper { - return &stringWrapper{ - s: s, - } -} - -func (s *stringWrapper) String() string { - return s.s -} diff --git a/dax/mds/controller/sets.go b/dax/mds/controller/sets.go index deb42a801..1a7313ec6 100644 --- a/dax/mds/controller/sets.go +++ b/dax/mds/controller/sets.go @@ -85,11 +85,11 @@ func (s TableSet) SortedSlice() dax.TableKeys { return ps } -func (s TableSet) QualifiedSortedSlice() map[dax.TableQualifier]dax.TableIDs { - m := make(map[dax.TableQualifier]dax.TableIDs) +func (s TableSet) QualifiedSortedSlice() map[dax.QualifiedDatabaseID]dax.TableIDs { + m := make(map[dax.QualifiedDatabaseID]dax.TableIDs) for p := range s { qtid := p.QualifiedTableID() - m[qtid.TableQualifier] = append(m[qtid.TableQualifier], qtid.ID) + m[qtid.QualifiedDatabaseID] = append(m[qtid.QualifiedDatabaseID], qtid.ID) } // Sort the slices in the map. diff --git a/dax/mds/controller/snapping_turtle.go b/dax/mds/controller/snapping_turtle.go index 0af9e2dd7..641555927 100644 --- a/dax/mds/controller/snapping_turtle.go +++ b/dax/mds/controller/snapping_turtle.go @@ -30,7 +30,29 @@ func (c *Controller) snappingTurtleRoutine(period time.Duration, control chan st func (c *Controller) snapAll() { c.logger.Debugf("TURTLE: snapAll") ctx := context.Background() - computeNodes, err := c.ComputeBalancer.CurrentState(ctx) + + tx, err := c.boltDB.BeginTx(ctx, false) + if err != nil { + c.logger.Printf("Error getting transaction for snapping turtle: %v", err) + return + } + defer tx.Rollback() + + qdbs, err := c.Schemar.Databases(tx, "") + if err != nil { + c.logger.Printf("couldn't get databases: %v", err) + } + + for _, qdb := range qdbs { + c.snapAllForDatabase(tx, qdb.QualifiedID()) + } + + c.logger.Debugf("TURTLE: snapAll complete") +} + +func (c *Controller) snapAllForDatabase(tx dax.Transaction, qdbid dax.QualifiedDatabaseID) { + c.logger.Debugf("TURTLE: snapAllForDatabase: %s", qdbid) + computeNodes, err := c.Balancer.CurrentState(tx, dax.RoleTypeCompute, qdbid) if err != nil { c.logger.Printf("Error getting compute balancer state for snapping turtle: %v", err) } @@ -52,15 +74,17 @@ func (c *Controller) snapAll() { if err != nil { c.logger.Printf("couldn't decode a shard out of the job: '%s', err: %v", workerInfo.Jobs[i], err) } - c.SnapshotShardData(ctx, j.t.QualifiedTableID(), j.shardNum()) + if err := c.snapshotShardData(tx, j.t.QualifiedTableID(), j.shardNum()); err != nil { + c.logger.Printf("Couldn't snapshot table: %s, shard: %d, error: %v", j.t, j.shardNum(), err) + } } i++ } // Get all tables across all orgs/dbs so we can snapshot all keyed - // fields and look up whether a table is keyed to snapshot it's + // fields and look up whether a table is keyed to snapshot its // partitions. - tables, err := c.Schemar.Tables(ctx, dax.TableQualifier{}) + tables, err := c.Schemar.Tables(tx, dax.QualifiedDatabaseID{}) if err != nil { c.logger.Printf("Couldn't get schema for snapshotting keys: %v", err) return @@ -71,8 +95,7 @@ func (c *Controller) snapAll() { tableMap[table.Key()] = table for _, f := range table.Fields { if f.StringKeys() && !f.IsPrimaryKey() { - err := c.SnapshotFieldKeys(ctx, table.QualifiedID(), f.Name) - if err != nil { + if err := c.snapshotFieldKeys(tx, table.QualifiedID(), f.Name); err != nil { c.logger.Printf("Couldn't snapshot table: %s, field: %s, error: %v", table, f.Name, err) } } @@ -83,7 +106,7 @@ func (c *Controller) snapAll() { // for any partition that goes with a keyed table. Doing the same // weird nested loop thing to avoid doing all jobs on one node // back to back. - translateNodes, err := c.TranslateBalancer.CurrentState(ctx) + translateNodes, err := c.Balancer.CurrentState(tx, dax.RoleTypeTranslate, qdbid) if err != nil { c.logger.Printf("Error getting translate balancer state for snapping turtle: %v", err) } @@ -101,12 +124,14 @@ func (c *Controller) snapAll() { if err != nil { table := tableMap[j.table()] if table.StringKeys() { - c.SnapshotTableKeys(ctx, table.QualifiedID(), j.partitionNum()) + if err := c.snapshotTableKeys(tx, table.QualifiedID(), j.partitionNum()); err != nil { + c.logger.Printf("Couldn't snapshot table: %s, partition: %d, error: %v", table, j.partitionNum(), err) + } } c.logger.Printf("couldn't decode a partition out of the job: '%s', err: %v", workerInfo.Jobs[i], err) } } i++ } - c.logger.Debugf("TURTLE: snapAll complete") + c.logger.Debugf("TURTLE: snapAllForDatabase complete: %s", qdbid) } diff --git a/dax/mds/controller/stringers.go b/dax/mds/controller/stringers.go index d47150470..6d56c8173 100644 --- a/dax/mds/controller/stringers.go +++ b/dax/mds/controller/stringers.go @@ -20,6 +20,10 @@ func (p pUnit) String() string { return fmt.Sprintf("%s|part_%d", p.t, p.p) } +func (p pUnit) Job() dax.Job { + return dax.Job(fmt.Sprintf("%s|part_%d", p.t, p.p)) +} + func (p pUnit) table() dax.TableKey { return p.t } @@ -72,6 +76,10 @@ func (s sUnit) String() string { return fmt.Sprintf("%s|shard_%s", s.t, s.s) } +func (s sUnit) Job() dax.Job { + return dax.Job(fmt.Sprintf("%s|shard_%s", s.t, s.s)) +} + func (s sUnit) table() dax.TableKey { return s.t } diff --git a/dax/mds/http/handler.go b/dax/mds/http/handler.go index 32b46caf9..9926416d2 100644 --- a/dax/mds/http/handler.go +++ b/dax/mds/http/handler.go @@ -18,6 +18,9 @@ func Handler(mds *mds.MDS) http.Handler { router.HandleFunc("/health", server.getHealth).Methods("GET").Name("GetHealth") // mds endpoints. + router.HandleFunc("/create-database", server.postCreateDatabase).Methods("POST").Name("PostCreateDatabase") + router.HandleFunc("/database-by-id", server.postDatabaseByID).Methods("POST").Name("PostDatabaseByID") + router.HandleFunc("/create-table", server.postCreateTable).Methods("POST").Name("PostCreateTable") router.HandleFunc("/drop-table", server.postDropTable).Methods("POST").Name("PostDropTable") router.HandleFunc("/create-field", server.postCreateField).Methods("POST").Name("PostCreateField") @@ -57,6 +60,55 @@ func (s *server) getHealth(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } +// POST /create-database +func (s *server) postCreateDatabase(w http.ResponseWriter, r *http.Request) { + body := r.Body + defer body.Close() + + ctx := r.Context() + + req := &dax.QualifiedDatabase{} + if err := json.NewDecoder(body).Decode(req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + err := s.mds.CreateDatabase(ctx, req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := json.NewEncoder(w).Encode(req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } +} + +// POST /database +func (s *server) postDatabaseByID(w http.ResponseWriter, r *http.Request) { + body := r.Body + defer body.Close() + + ctx := r.Context() + + qdbid := dax.QualifiedDatabaseID{} + if err := json.NewDecoder(body).Decode(&qdbid); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + resp, err := s.mds.DatabaseByID(ctx, qdbid) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if err := json.NewEncoder(w).Encode(resp); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } +} + // POST /create-table func (s *server) postCreateTable(w http.ResponseWriter, r *http.Request) { body := r.Body @@ -76,15 +128,12 @@ func (s *server) postCreateTable(w http.ResponseWriter, r *http.Request) { return } - resp := CreateTableResponse(*req) - if err := json.NewEncoder(w).Encode(resp); err != nil { + if err := json.NewEncoder(w).Encode(req); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } } -type CreateTableResponse dax.QualifiedTable - // POST /table func (s *server) postTable(w http.ResponseWriter, r *http.Request) { body := r.Body @@ -122,7 +171,7 @@ func (s *server) postTableID(w http.ResponseWriter, r *http.Request) { return } - qtid, err := s.mds.TableID(ctx, req.TableQualifier, req.Name) + qtid, err := s.mds.TableID(ctx, req.QualifiedDatabaseID, req.Name) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -221,10 +270,10 @@ func (s *server) postTables(w http.ResponseWriter, r *http.Request) { return } - qual := dax.NewTableQualifier(req.OrganizationID, req.DatabaseID) + qdbid := dax.NewQualifiedDatabaseID(req.OrganizationID, req.DatabaseID) ids := req.TableIDs - resp, err := s.mds.Tables(ctx, qual, ids...) + resp, err := s.mds.Tables(ctx, qdbid, ids...) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return diff --git a/dax/mds/mds.go b/dax/mds/mds.go index 666fda0ce..ffa64e441 100644 --- a/dax/mds/mds.go +++ b/dax/mds/mds.go @@ -3,7 +3,6 @@ package mds import ( "context" - "fmt" "os" "sync" "time" @@ -57,7 +56,6 @@ type MDS struct { // we need to be sure to close the boltDBs that are created in mds.New() // whenever mds.Close() is called. These are pointers to those DBs so we can // close them. - schemarDB *boltdb.DB controllerDB *boltdb.DB logger logger.Logger @@ -88,25 +86,20 @@ func New(cfg Config) *MDS { logr.Warnf("no DataDir given (like '/path/to/directory') using temp dir at '%s'", cfg.DataDir) } - schemarDB, err := boltdb.NewSvcBolt(cfg.DataDir, "schemar", schemarboltdb.SchemarBuckets...) + buckets := append(schemarboltdb.SchemarBuckets, balancerboltdb.BalancerBuckets...) + controllerDB, err := boltdb.NewSvcBolt(cfg.DataDir, "controller", buckets...) if err != nil { - logr.Printf("Error creating schemar db: %v", err) + logr.Printf(errors.Wrap(err, "creating controller bolt").Error()) os.Exit(1) } - schemar := schemarboltdb.NewSchemar(schemarDB, logr) - - controllerDB, err := boltdb.NewSvcBolt(cfg.DataDir, "balancer", naiveboltdb.NaiveBalancerBuckets...) - if err != nil { - logr.Printf(errors.Wrap(err, "creating balancer bolt").Error()) - os.Exit(1) - } + schemar := schemarboltdb.NewSchemar(controllerDB, logr) controllerCfg := controller.Config{ - Director: cfg.Director, - Schemar: schemar, - ComputeBalancer: naiveboltdb.NewBalancer("compute", controllerDB, logr), - TranslateBalancer: naiveboltdb.NewBalancer("translate", controllerDB, logr), + Director: cfg.Director, + Schemar: schemar, + + Balancer: balancerboltdb.NewBalancer(controllerDB, schemar, logr), RegistrationBatchTimeout: cfg.RegistrationBatchTimeout, SnappingTurtleTimeout: cfg.SnappingTurtleTimeout, @@ -140,7 +133,6 @@ func New(cfg Config) *MDS { poller: poller, schemar: schemar, - schemarDB: schemarDB, controllerDB: controllerDB, logger: logr, @@ -169,9 +161,6 @@ func (m *MDS) Stop() error { m.poller.Stop() m.controller.Stop() - if m.schemarDB != nil { - m.schemarDB.Close() - } if m.controllerDB != nil { m.controllerDB.Close() } @@ -179,38 +168,17 @@ func (m *MDS) Stop() error { return nil } -// sanitizeQTID populates Table.ID (by looking up the table, by name, in -// schemar) for a given table having only a Name value, but no ID. -func (m *MDS) sanitizeQTID(ctx context.Context, qtid *dax.QualifiedTableID) error { - if qtid.ID == "" { - nqtid, err := m.schemar.TableID(ctx, qtid.TableQualifier, qtid.Name) - if err != nil { - return errors.Wrap(err, "getting table ID") - } - qtid.ID = nqtid.ID - } - return nil +// CreateDatabase handles a create table request. +func (m *MDS) CreateDatabase(ctx context.Context, qdb *dax.QualifiedDatabase) error { + return m.controller.CreateDatabase(ctx, qdb) +} + +func (m *MDS) DatabaseByID(ctx context.Context, qdbid dax.QualifiedDatabaseID) (*dax.QualifiedDatabase, error) { + return m.controller.DatabaseByID(ctx, qdbid) } // CreateTable handles a create table request. func (m *MDS) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) error { - m.mu.Lock() - defer m.mu.Unlock() - - // Create Table ID. - if _, err := qtbl.CreateID(); err != nil { - return errors.Wrap(err, "creating table ID") - } - - // Create the table in schemar. - if err := m.schemar.CreateTable(ctx, qtbl); err != nil { - return errors.Wrapf(err, "creating table: %s", qtbl) - } - - // TODO: if error here, we should probably roll-back the - // schemar.CreateTable() request. - - // Add the table to the controller. return m.controller.CreateTable(ctx, qtbl) } @@ -218,216 +186,61 @@ func (m *MDS) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) error { // reason about consistency here? What if controller DropTable // succeeds, but schemar fails? func (m *MDS) DropTable(ctx context.Context, qtid dax.QualifiedTableID) error { - m.mu.Lock() - defer m.mu.Unlock() - - if err := m.sanitizeQTID(ctx, &qtid); err != nil { - return errors.Wrap(err, "sanitizing") - } - - if err := m.controller.DropTable(ctx, qtid); err != nil { - return errors.Wrapf(err, "dropping table: %s", qtid) - } - - return m.schemar.DropTable(ctx, qtid) -} - -type CreateFieldRequest struct { - Table dax.TableName - Field *dax.Field + return m.controller.DropTable(ctx, qtid) } // CreateField handles a create Field request. func (m *MDS) CreateField(ctx context.Context, qtid dax.QualifiedTableID, fld *dax.Field) error { - m.mu.Lock() - defer m.mu.Unlock() - - if err := m.sanitizeQTID(ctx, &qtid); err != nil { - return errors.Wrap(err, "sanitizing") - } - - // Create the field in schemar. - if err := m.schemar.CreateField(ctx, qtid, fld); err != nil { - return errors.Wrapf(err, "creating field: %s, %s", qtid, fld) - } - - // Add the table to the controller. return m.controller.CreateField(ctx, qtid, fld) } // DropField handles a drop Field request. func (m *MDS) DropField(ctx context.Context, qtid dax.QualifiedTableID, fldName dax.FieldName) error { - m.mu.Lock() - defer m.mu.Unlock() - - if err := m.sanitizeQTID(ctx, &qtid); err != nil { - return errors.Wrap(err, "sanitizing") - } - - // Drop the field from schemar. - if err := m.schemar.DropField(ctx, qtid, fldName); err != nil { - return errors.Wrapf(err, "dropping field: %s, %s", qtid, fldName) - } - - // Drop the field from the controller. return m.controller.DropField(ctx, qtid, fldName) } -type DropFieldResponse struct{} - // Table handles a table request. func (m *MDS) Table(ctx context.Context, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - if err := m.sanitizeQTID(ctx, &qtid); err != nil { - return nil, errors.Wrap(err, "sanitizing") - } - - return m.schemar.Table(ctx, qtid) + return m.controller.Table(ctx, qtid) } // Tables handles a tables request. -func (m *MDS) Tables(ctx context.Context, qual dax.TableQualifier, ids ...dax.TableID) ([]*dax.QualifiedTable, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - return m.schemar.Tables(ctx, qual, ids...) +func (m *MDS) Tables(ctx context.Context, qdbid dax.QualifiedDatabaseID, ids ...dax.TableID) ([]*dax.QualifiedTable, error) { + return m.controller.Tables(ctx, qdbid, ids...) } // TableID handles a table id (i.e. by name) request. -func (m *MDS) TableID(ctx context.Context, qual dax.TableQualifier, name dax.TableName) (dax.QualifiedTableID, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - return m.schemar.TableID(ctx, qual, name) +func (m *MDS) TableID(ctx context.Context, qdbid dax.QualifiedDatabaseID, name dax.TableName) (dax.QualifiedTableID, error) { + return m.controller.TableID(ctx, qdbid, name) } // IngestPartition handles an ingest partition request. func (m *MDS) IngestPartition(ctx context.Context, qtid dax.QualifiedTableID, partnNum dax.PartitionNum) (dax.Address, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - if err := m.sanitizeQTID(ctx, &qtid); err != nil { - return "", errors.Wrap(err, "sanitizing") - } - - // Verify that the table exists. - if _, err := m.schemar.Table(ctx, qtid); err != nil { - return "", err - } - - partitions := dax.PartitionNums{partnNum} - - nodes, err := m.controller.TranslateNodes(ctx, qtid, partitions, true) - if err != nil { - return "", err - } - - if l := len(nodes); l == 0 { - return "", controller.NewErrNoAvailableNode() - } else if l > 1 { - return "", controller.NewErrInternal( - fmt.Sprintf("unexpected number of nodes: %d", l)) - } - - node := nodes[0] - - // Verify that the node returned is actually responsible for the partition - // requested. - if node.Table != qtid.Key() { - return "", controller.NewErrInternal( - fmt.Sprintf("table returned (%s) does not match requested (%s)", node.Table, qtid)) - } else if l := len(node.Partitions); l != 1 { - return "", controller.NewErrInternal( - fmt.Sprintf("unexpected number of partitions returned: %d", l)) - } else if p := node.Partitions[0]; p != partnNum { - return "", controller.NewErrInternal( - fmt.Sprintf("partition returned (%d) does not match requested (%d)", p, partnNum)) - } - - return node.Address, nil + return m.controller.IngestPartition(ctx, qtid, partnNum) } // IngestShard handles an ingest shard request. func (m *MDS) IngestShard(ctx context.Context, qtid dax.QualifiedTableID, shrdNum dax.ShardNum) (dax.Address, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - if err := m.sanitizeQTID(ctx, &qtid); err != nil { - return "", errors.Wrap(err, "sanitizing") - } - - // Verify that the table exists. - if _, err := m.schemar.Table(ctx, qtid); err != nil { - return "", err - } - - shards := dax.ShardNums{shrdNum} - - nodes, err := m.controller.ComputeNodes(ctx, qtid, shards, true) - if err != nil { - return "", err - } - - if l := len(nodes); l == 0 { - return "", controller.NewErrNoAvailableNode() - } else if l > 1 { - return "", controller.NewErrInternal( - fmt.Sprintf("unexpected number of nodes: %d", l)) - } - - node := nodes[0] - - // Verify that the node returned is actually responsible for the shard - // requested. - if node.Table != qtid.Key() { - return "", controller.NewErrInternal( - fmt.Sprintf("table returned (%s) does not match requested (%s)", node.Table, qtid)) - } else if l := len(node.Shards); l != 1 { - return "", controller.NewErrInternal( - fmt.Sprintf("unexpected number of shards returned: %d", l)) - } else if s := node.Shards[0]; s != shrdNum { - return "", controller.NewErrInternal( - fmt.Sprintf("shard returned (%d) does not match requested (%d)", s, shrdNum)) - } - - return node.Address, nil + return m.controller.IngestShard(ctx, qtid, shrdNum) } // SnapshotTable handles a snapshot table request. func (m *MDS) SnapshotTable(ctx context.Context, qtid dax.QualifiedTableID) error { - if err := m.sanitizeQTID(ctx, &qtid); err != nil { - return errors.Wrap(err, "sanitizing") - } - return m.controller.SnapshotTable(ctx, qtid) } // SnapshotShardData handles a snapshot shard request. func (m *MDS) SnapshotShardData(ctx context.Context, qtid dax.QualifiedTableID, shardNum dax.ShardNum) error { - if err := m.sanitizeQTID(ctx, &qtid); err != nil { - return errors.Wrap(err, "sanitizing") - } - return m.controller.SnapshotShardData(ctx, qtid, shardNum) } // SnapshotTableKeys handles a snapshot table/keys request. func (m *MDS) SnapshotTableKeys(ctx context.Context, qtid dax.QualifiedTableID, partitionNum dax.PartitionNum) error { - if err := m.sanitizeQTID(ctx, &qtid); err != nil { - return errors.Wrap(err, "sanitizing") - } - return m.controller.SnapshotTableKeys(ctx, qtid, partitionNum) } // SnapshotFieldKeys handles a snapshot field/keys request. func (m *MDS) SnapshotFieldKeys(ctx context.Context, qtid dax.QualifiedTableID, fldName dax.FieldName) error { - if err := m.sanitizeQTID(ctx, &qtid); err != nil { - return errors.Wrap(err, "sanitizing") - } - return m.controller.SnapshotFieldKeys(ctx, qtid, fldName) } @@ -466,11 +279,7 @@ func (m *MDS) DeregisterNodes(ctx context.Context, addrs ...dax.Address) error { // ComputeNodes gets the compute nodes responsible for the table/shards // specified in the ComputeNodeRequest. func (m *MDS) ComputeNodes(ctx context.Context, qtid dax.QualifiedTableID, shardNums ...dax.ShardNum) ([]dax.ComputeNode, error) { - if err := m.sanitizeQTID(ctx, &qtid); err != nil { - return nil, errors.Wrap(err, "sanitizing") - } - - return m.controller.ComputeNodes(ctx, qtid, shardNums, false) + return m.controller.ComputeNodes(ctx, qtid, shardNums) } func (m *MDS) DebugNodes(ctx context.Context) ([]*dax.Node, error) { @@ -480,9 +289,5 @@ func (m *MDS) DebugNodes(ctx context.Context) ([]*dax.Node, error) { // TranslateNodes gets the translate nodes responsible for the table/partitions // specified in the TranslateNodeRequest. func (m *MDS) TranslateNodes(ctx context.Context, qtid dax.QualifiedTableID, partitionNums ...dax.PartitionNum) ([]dax.TranslateNode, error) { - if err := m.sanitizeQTID(ctx, &qtid); err != nil { - return nil, errors.Wrap(err, "sanitizing") - } - - return m.controller.TranslateNodes(ctx, qtid, partitionNums, false) + return m.controller.TranslateNodes(ctx, qtid, partitionNums) } diff --git a/dax/mds/schemar/boltdb/schemar.go b/dax/mds/schemar/boltdb/schemar.go index 14f3c433e..47c9951fd 100644 --- a/dax/mds/schemar/boltdb/schemar.go +++ b/dax/mds/schemar/boltdb/schemar.go @@ -4,7 +4,6 @@ package boltdb import ( "bytes" - "context" "encoding/json" "fmt" "strings" @@ -44,9 +43,224 @@ func NewSchemar(db *boltdb.DB, logger logger.Logger) *Schemar { } } +// CreateDatabase creates the database provided. If a database with the same +// name already exists then an error is returned. For now, we are not going to +// store the tables in the schemar Database struct. +func (s *Schemar) CreateDatabase(tx dax.Transaction, qdb *dax.QualifiedDatabase) error { + // Ensure the database id is not blank. + if qdb.ID == "" { + return schemar.NewErrDatabaseIDInvalid(qdb.ID) + } + + // Ensure the database name is not blank. + if qdb.Name == "" { + return schemar.NewErrDatabaseNameInvalid(qdb.Name) + } + + // Set the CreateAt value for the database. + // TODO(tlt): We may want to consider erroring here if the value is != 0. + if qdb.CreatedAt == 0 { + now := timestamp() + qdb.CreatedAt = now + } + + //////////// end validation + + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + // Ensure a database with that ID doesn't already exist. + if db, _ := s.databaseByID(txx, qdb.OrganizationID, qdb.ID); db != nil { + return dax.NewErrDatabaseIDExists(qdb.QualifiedID()) + } + + if err := s.putDatabase(txx, qdb); err != nil { + return errors.Wrap(err, "putting database") + } + + // In addition to storing the database in databaseKey, we want to store a + // reverse-lookup (i.e. index) on database name to the databaseKey. + if err := s.putDatabaseName(txx, qdb); err != nil { + return errors.Wrap(err, "putting database name") + } + + return nil +} + +func (s *Schemar) DatabaseByID(tx dax.Transaction, qdbid dax.QualifiedDatabaseID) (*dax.QualifiedDatabase, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() + } + + return s.databaseByID(txx, qdbid.OrganizationID, qdbid.DatabaseID) +} + +func (s *Schemar) databaseByID(tx *boltdb.Tx, orgID dax.OrganizationID, id dax.DatabaseID) (*dax.QualifiedDatabase, error) { + bkt := tx.Bucket(bucketSchemar) + if bkt == nil { + return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketSchemar) + } + + b := bkt.Get(databaseKey(orgID, id)) + if b == nil { + return nil, dax.NewErrDatabaseIDDoesNotExist(dax.QualifiedDatabaseID{OrganizationID: orgID, DatabaseID: id}) + } + + database := &dax.QualifiedDatabase{} + if err := json.Unmarshal(b, database); err != nil { + return nil, errors.Wrap(err, "unmarshalling database json") + } + + return database, nil +} + +func (s *Schemar) putDatabase(tx *boltdb.Tx, qdb *dax.QualifiedDatabase) error { + bkt := tx.Bucket(bucketSchemar) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketSchemar) + } + + val, err := json.Marshal(qdb) + if err != nil { + return errors.Wrap(err, "marshalling database to json") + } + + return bkt.Put(databaseKey(qdb.OrganizationID, qdb.ID), val) +} + +func (s *Schemar) putDatabaseName(tx *boltdb.Tx, qdb *dax.QualifiedDatabase) error { + bkt := tx.Bucket(bucketSchemar) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketSchemar) + } + + return bkt.Put(databaseNameKey(qdb.OrganizationID, qdb.Name), databaseKey(qdb.OrganizationID, qdb.ID)) +} + +// DropDatabase drops the given database. If the named/IDed database does not +// exist then an error is returned. +func (s *Schemar) DropDatabase(tx dax.Transaction, qdbid dax.QualifiedDatabaseID) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + // Ensure the database exists. + qdb, err := s.databaseByID(txx, qdbid.OrganizationID, qdbid.DatabaseID) + if err != nil { + return errors.Wrap(err, "getting database by id") + } + + bkt := txx.Bucket(bucketSchemar) + if bkt == nil { + return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketSchemar) + } + + // Delete the database by ID. + if err := bkt.Delete(databaseKey(qdb.OrganizationID, qdb.ID)); err != nil { + return errors.Wrap(err, "deleting database by id") + } + + // Delete the reverse-lookup database by Name. + if err := bkt.Delete(databaseNameKey(qdb.OrganizationID, qdb.Name)); err != nil { + return errors.Wrap(err, "deleting database by name") + } + + return nil +} + +// SetDatabaseOptions overwrites the existing database options with those +// provided for the given database. +func (s *Schemar) SetDatabaseOptions(tx dax.Transaction, qdbid dax.QualifiedDatabaseID, opts dax.DatabaseOptions) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + // Get the database. + qdb, err := s.databaseByID(txx, qdbid.OrganizationID, qdbid.DatabaseID) + if err != nil { + return errors.Wrapf(err, "getting database: %s", qdbid) + } + + // Set the new options. + qdb.Options = opts + + // Put the database. + if err := s.putDatabase(txx, qdb); err != nil { + return errors.Wrap(err, "putting database") + } + + return nil +} + +func (s *Schemar) Databases(tx dax.Transaction, orgID dax.OrganizationID, ids ...dax.DatabaseID) ([]*dax.QualifiedDatabase, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() + } + + return s.getDatabases(txx, orgID, ids...) +} + +func (s *Schemar) getDatabases(tx *boltdb.Tx, orgID dax.OrganizationID, ids ...dax.DatabaseID) (dax.QualifiedDatabases, error) { + c := tx.Bucket(bucketSchemar).Cursor() + + // Deserialize rows into Database objects. + databases := make(dax.QualifiedDatabases, 0) + + var filterByID bool + if len(ids) > 0 { + filterByID = true + } + + prefix := []byte(fmt.Sprintf(prefixFmtDatabases, orgID)) + if orgID == "" { + prefix = []byte(prefixDatabases) + } + + for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { + if v == nil { + s.logger.Printf("nil value for key: %s", k) + continue + } + + dbID, err := keyDatabaseID(k) + if err != nil { + return nil, errors.Wrap(err, "getting database from key") + } + + // Only include databases provided in the ids filter. + if filterByID && !containsDatabaseID(ids, dbID) { + continue + } + + database := &dax.QualifiedDatabase{} + if err := json.Unmarshal(v, database); err != nil { + return nil, errors.Wrap(err, "unmarshalling database json") + } + + databases = append(databases, database) + } + + return databases, nil +} + +func containsDatabaseID(s []dax.DatabaseID, e dax.DatabaseID) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} + // CreateTable creates the table provided. If a table with the same name already // exists then an error is returned. -func (s *Schemar) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) error { +func (s *Schemar) CreateTable(tx dax.Transaction, qtbl *dax.QualifiedTable) error { // Ensure the table id is not blank. if qtbl.ID == "" { return schemar.NewErrTableIDInvalid(qtbl.ID) @@ -76,33 +290,37 @@ func (s *Schemar) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) err //////////// end validation - tx, err := s.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "getting transaction") + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() + } + + // Ensure the database, defined in the table's QualifiedDatabaseID, exists. + if _, err := s.databaseByID(txx, qtbl.OrganizationID, qtbl.DatabaseID); err != nil { + return errors.Wrap(err, "validating database") } - defer tx.Rollback() // Ensure a table with that ID doesn't already exist. - if t, _ := s.tableByID(tx, qtbl.TableQualifier, qtbl.ID); t != nil { + if t, _ := s.tableByID(txx, qtbl.QualifiedDatabaseID, qtbl.ID); t != nil { return dax.NewErrTableIDExists(qtbl.QualifiedID()) } - if err := s.putTable(tx, qtbl); err != nil { + if err := s.putTable(txx, qtbl); err != nil { return errors.Wrap(err, "putting table") } // In addition to storing the table in tableKey, we want to store a reverse-lookup // (i.e. index) on table name to the tableKey. - if err := s.putTableName(tx, qtbl); err != nil { + if err := s.putTableName(txx, qtbl); err != nil { return errors.Wrap(err, "putting table name") } - return tx.Commit() + return nil } // CreateField creates the field provided in the given table. If a field with // the same name already exists then an error is returned. -func (s *Schemar) CreateField(ctx context.Context, qtid dax.QualifiedTableID, fld *dax.Field) error { +func (s *Schemar) CreateField(tx dax.Transaction, qtid dax.QualifiedTableID, fld *dax.Field) error { // Ensure the field name is not blank. if fld.Name == "" { return schemar.NewErrFieldNameInvalid(fld.Name) @@ -110,14 +328,13 @@ func (s *Schemar) CreateField(ctx context.Context, qtid dax.QualifiedTableID, fl //////////// end validation - tx, err := s.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "getting transaction") + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() } - defer tx.Rollback() // Get the table. - qtbl, err := s.tableByQTID(tx, qtid) + qtbl, err := s.tableByQTID(txx, qtid) if err != nil { return errors.Wrap(err, "getting table by id") } @@ -130,23 +347,22 @@ func (s *Schemar) CreateField(ctx context.Context, qtid dax.QualifiedTableID, fl qtbl.Fields = append(qtbl.Fields, fld) // Write table back to database. - if err := s.putTable(tx, qtbl); err != nil { + if err := s.putTable(txx, qtbl); err != nil { return errors.Wrap(err, "putting table") } - return tx.Commit() + return nil } // DropField removes the field from the table. -func (s *Schemar) DropField(ctx context.Context, qtid dax.QualifiedTableID, fldName dax.FieldName) error { - tx, err := s.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "getting transaction") +func (s *Schemar) DropField(tx dax.Transaction, qtid dax.QualifiedTableID, fldName dax.FieldName) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() } - defer tx.Rollback() // Get the table. - qtbl, err := s.tableByQTID(tx, qtid) + qtbl, err := s.tableByQTID(txx, qtid) if err != nil { return errors.Wrap(err, "getting table by id") } @@ -159,11 +375,11 @@ func (s *Schemar) DropField(ctx context.Context, qtid dax.QualifiedTableID, fldN _ = qtbl.RemoveField(fldName) // Write table back to database. - if err := s.putTable(tx, qtbl); err != nil { + if err := s.putTable(txx, qtbl); err != nil { return errors.Wrap(err, "putting table") } - return tx.Commit() + return nil } func (s *Schemar) putTable(tx *boltdb.Tx, qtbl *dax.QualifiedTable) error { @@ -191,43 +407,43 @@ func (s *Schemar) putTableName(tx *boltdb.Tx, qtbl *dax.QualifiedTable) error { // Table returns the TableInfo for the given table. An error is returned if the // table does not exist. -func (s *Schemar) Table(ctx context.Context, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrap(err, "beginning tx") +func (s *Schemar) Table(tx dax.Transaction, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() } - defer tx.Rollback() - return s.tableByQTID(tx, qtid) + return s.tableByQTID(txx, qtid) } -// tableByQTID gets the full qualified table by the QualifiedTableID whether it has Name or ID set. +// tableByQTID gets the full qualified table by the QualifiedTableID whether it +// has Name or ID set. func (s *Schemar) tableByQTID(tx *boltdb.Tx, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) { if qtid.ID == "" { - return s.tableByName(tx, qtid.TableQualifier, qtid.Name) + return s.tableByName(tx, qtid.QualifiedDatabaseID, qtid.Name) } - return s.tableByID(tx, qtid.TableQualifier, qtid.ID) + return s.tableByID(tx, qtid.QualifiedDatabaseID, qtid.ID) } -func (s *Schemar) tableByName(tx *boltdb.Tx, qual dax.TableQualifier, name dax.TableName) (*dax.QualifiedTable, error) { - qtid, err := s.tableIDByName(tx, qual, name) +func (s *Schemar) tableByName(tx *boltdb.Tx, qdbid dax.QualifiedDatabaseID, name dax.TableName) (*dax.QualifiedTable, error) { + qtid, err := s.tableIDByName(tx, qdbid, name) if err != nil { return nil, errors.Wrap(err, "getting table ID") } - return s.tableByID(tx, qtid.TableQualifier, qtid.ID) // TODO remove? + return s.tableByID(tx, qtid.QualifiedDatabaseID, qtid.ID) // TODO remove? } -func (s *Schemar) tableByID(tx *boltdb.Tx, qual dax.TableQualifier, id dax.TableID) (*dax.QualifiedTable, error) { +func (s *Schemar) tableByID(tx *boltdb.Tx, qdbid dax.QualifiedDatabaseID, id dax.TableID) (*dax.QualifiedTable, error) { bkt := tx.Bucket(bucketSchemar) if bkt == nil { return nil, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketSchemar) } - b := bkt.Get(tableKey(qual.OrganizationID, qual.DatabaseID, id)) + b := bkt.Get(tableKey(qdbid.OrganizationID, qdbid.DatabaseID, id)) if b == nil { - return nil, dax.NewErrTableIDDoesNotExist(dax.QualifiedTableID{TableQualifier: qual, ID: id}) + return nil, dax.NewErrTableIDDoesNotExist(dax.QualifiedTableID{QualifiedDatabaseID: qdbid, ID: id}) } table := &dax.QualifiedTable{} @@ -238,13 +454,13 @@ func (s *Schemar) tableByID(tx *boltdb.Tx, qual dax.TableQualifier, id dax.Table return table, nil } -func (s *Schemar) tableIDByName(tx *boltdb.Tx, qual dax.TableQualifier, name dax.TableName) (dax.QualifiedTableID, error) { +func (s *Schemar) tableIDByName(tx *boltdb.Tx, qdbid dax.QualifiedDatabaseID, name dax.TableName) (dax.QualifiedTableID, error) { bkt := tx.Bucket(bucketSchemar) if bkt == nil { return dax.QualifiedTableID{}, errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketSchemar) } - b := bkt.Get(tableNameKey(qual.OrganizationID, qual.DatabaseID, name)) + b := bkt.Get(tableNameKey(qdbid.OrganizationID, qdbid.DatabaseID, name)) if b == nil { return dax.QualifiedTableID{}, dax.NewErrTableNameDoesNotExist(name) } @@ -254,17 +470,16 @@ func (s *Schemar) tableIDByName(tx *boltdb.Tx, qual dax.TableQualifier, name dax // Tables returns a list of Table for all existing tables. If one or more table // IDs is provided, then only those will be included in the output. -func (s *Schemar) Tables(ctx context.Context, qual dax.TableQualifier, ids ...dax.TableID) ([]*dax.QualifiedTable, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return nil, errors.Wrap(err, "beginning tx") +func (s *Schemar) Tables(tx dax.Transaction, qdbid dax.QualifiedDatabaseID, ids ...dax.TableID) ([]*dax.QualifiedTable, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return nil, dax.NewErrInvalidTransaction() } - defer tx.Rollback() - return s.getTables(ctx, tx, qual, ids...) + return s.getTables(txx, qdbid, ids...) } -func (s *Schemar) getTables(ctx context.Context, tx *boltdb.Tx, qual dax.TableQualifier, ids ...dax.TableID) (dax.QualifiedTables, error) { +func (s *Schemar) getTables(tx *boltdb.Tx, qdbid dax.QualifiedDatabaseID, ids ...dax.TableID) (dax.QualifiedTables, error) { c := tx.Bucket(bucketSchemar).Cursor() // Deserialize rows into Table objects. @@ -275,11 +490,11 @@ func (s *Schemar) getTables(ctx context.Context, tx *boltdb.Tx, qual dax.TableQu filterByID = true } - prefix := []byte(fmt.Sprintf(prefixFmtTables, qual.OrganizationID, qual.DatabaseID)) - if qual.OrganizationID == "" && qual.DatabaseID == "" { + prefix := []byte(fmt.Sprintf(prefixFmtTables, qdbid.OrganizationID, qdbid.DatabaseID)) + if qdbid.OrganizationID == "" && qdbid.DatabaseID == "" { prefix = []byte(prefixTables) - } else if qual.DatabaseID == "" { - prefix = []byte(fmt.Sprintf(prefixFmtTablesOrg, qual.OrganizationID)) + } else if qdbid.DatabaseID == "" { + prefix = []byte(fmt.Sprintf(prefixFmtTablesOrg, qdbid.OrganizationID)) } for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { @@ -320,20 +535,19 @@ func containsTableID(s []dax.TableID, e dax.TableID) bool { // DropTable drops the given table. If the named/IDed table does not exist // then an error is returned. -func (s *Schemar) DropTable(ctx context.Context, qtid dax.QualifiedTableID) error { - tx, err := s.db.BeginTx(ctx, true) - if err != nil { - return errors.Wrap(err, "getting transaction") +func (s *Schemar) DropTable(tx dax.Transaction, qtid dax.QualifiedTableID) error { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.NewErrInvalidTransaction() } - defer tx.Rollback() // Ensure the table exists. - qtbl, err := s.tableByQTID(tx, qtid) + qtbl, err := s.tableByQTID(txx, qtid) if err != nil { return errors.Wrap(err, "getting table by id") } - bkt := tx.Bucket(bucketSchemar) + bkt := txx.Bucket(bucketSchemar) if bkt == nil { return errors.Errorf(boltdb.ErrFmtBucketNotFound, bucketSchemar) } @@ -348,16 +562,52 @@ func (s *Schemar) DropTable(ctx context.Context, qtid dax.QualifiedTableID) erro return errors.Wrap(err, "deleting table by name") } - return tx.Commit() + return nil +} + +func (s *Schemar) TableID(tx dax.Transaction, qdbid dax.QualifiedDatabaseID, name dax.TableName) (dax.QualifiedTableID, error) { + txx, ok := tx.(*boltdb.Tx) + if !ok { + return dax.QualifiedTableID{}, dax.NewErrInvalidTransaction() + } + + return s.tableIDByName(txx, qdbid, name) } const ( prefixTables = "tables/" - prefixFmtTablesOrg = prefixTables + "%s/" - prefixFmtTables = prefixFmtTablesOrg + "%s/" - prefixFmtTableNames = "tablenames/%s/%s/" + prefixFmtTablesOrg = prefixTables + "%s/" // org-id + prefixFmtTables = prefixFmtTablesOrg + "%s/" // db-id + prefixFmtTableNames = "tablenames/%s/%s/" // org-id, db-id + + prefixDatabases = "databases/" + prefixFmtDatabases = prefixDatabases + "%s/" // org-id + prefixFmtDatabase = prefixFmtDatabases + "%s" // db-id + prefixFmtDatabaseNames = "databasenames/%s/" // org-id ) +// databaseKey returns a key based on a qualified database ID. +func databaseKey(orgID dax.OrganizationID, dbID dax.DatabaseID) []byte { + key := fmt.Sprintf(prefixFmtDatabase, orgID, dbID) + return []byte(key) +} + +// databaseNameKey returns a key based on a qualified database name. +func databaseNameKey(orgID dax.OrganizationID, name dax.DatabaseName) []byte { + key := fmt.Sprintf(prefixFmtDatabaseNames+"%s", orgID, name) + return []byte(key) +} + +// keyDatabaseID gets the DatabaseID out of the key. +func keyDatabaseID(key []byte) (dax.DatabaseID, error) { + parts := strings.Split(string(key), "/") + if len(parts) != 3 { + return "", errors.New(errors.ErrUncoded, "database key format expected: `databases/orgID/dbID`") + } + + return dax.DatabaseID(parts[2]), nil +} + // tableKey returns a key based on a qualified table ID. func tableKey(orgID dax.OrganizationID, dbID dax.DatabaseID, tblID dax.TableID) []byte { key := fmt.Sprintf(prefixFmtTables+"%s", orgID, dbID, tblID) @@ -388,7 +638,7 @@ func keyQualifiedTableID(key []byte) (dax.QualifiedTableID, error) { } return dax.NewQualifiedTableID( - dax.NewTableQualifier( + dax.NewQualifiedDatabaseID( dax.OrganizationID(parts[1]), dax.DatabaseID(parts[2]), ), @@ -396,16 +646,6 @@ func keyQualifiedTableID(key []byte) (dax.QualifiedTableID, error) { ), nil } -func (s *Schemar) TableID(ctx context.Context, qual dax.TableQualifier, name dax.TableName) (dax.QualifiedTableID, error) { - tx, err := s.db.BeginTx(ctx, false) - if err != nil { - return dax.QualifiedTableID{}, err - } - defer tx.Rollback() - - return s.tableIDByName(tx, qual, name) -} - func timestamp() int64 { return time.Now().UnixNano() } diff --git a/dax/mds/schemar/boltdb/schemar_test.go b/dax/mds/schemar/boltdb/schemar_test.go index 6af10d7bd..20a1d4e27 100644 --- a/dax/mds/schemar/boltdb/schemar_test.go +++ b/dax/mds/schemar/boltdb/schemar_test.go @@ -16,6 +16,7 @@ import ( func TestSchemar(t *testing.T) { orgID := dax.OrganizationID("acme") dbID := dax.DatabaseID("db1") + dbName := dax.DatabaseName("dbname1") invalidTableID := dax.TableID("invalidID") tableName := dax.TableName("foo") tableName0 := dax.TableName("foo") @@ -25,21 +26,36 @@ func TestSchemar(t *testing.T) { partitionN := 12 ctx := context.Background() - qual := dax.NewTableQualifier(orgID, dbID) + qdbid := dax.NewQualifiedDatabaseID(orgID, dbID) - db := testbolt.MustOpenDB(t) - defer testbolt.MustCloseDB(t, db) - - t.Cleanup(func() { - testbolt.CleanupDB(t, db.Path()) - }) - - // Initialize the buckets. - assert.NoError(t, db.InitializeBuckets(boltdb.SchemarBuckets...)) + qdb := &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: dax.Database{ + ID: dbID, + Name: dbName, + }, + } t.Run("NewSchemar", func(t *testing.T) { + db := testbolt.MustOpenDB(t) + defer testbolt.MustCloseDB(t, db) + + t.Cleanup(func() { + testbolt.CleanupDB(t, db.Path()) + }) + + // Initialize the buckets. + assert.NoError(t, db.InitializeBuckets(boltdb.SchemarBuckets...)) + s := boltdb.NewSchemar(db, logger.NopLogger) + tx, err := db.BeginTx(ctx, true) + assert.NoError(t, err) + defer tx.Rollback() + + // Create database. + assert.NoError(t, s.CreateDatabase(tx, qdb)) + // Add new table. tbl := dax.NewTable(tableName) tbl.CreateID() @@ -53,11 +69,11 @@ func TestSchemar(t *testing.T) { Type: dax.BaseTypeInt, }, } - qtbl := dax.NewQualifiedTable(qual, tbl) - assert.NoError(t, s.CreateTable(ctx, qtbl)) + qtbl := dax.NewQualifiedTable(qdbid, tbl) + assert.NoError(t, s.CreateTable(tx, qtbl)) // Try adding the table again. - err := s.CreateTable(ctx, qtbl) + err = s.CreateTable(tx, qtbl) if assert.Error(t, err) { assert.True(t, errors.Is(err, dax.ErrTableIDExists)) } @@ -66,32 +82,32 @@ func TestSchemar(t *testing.T) { // Get the table. { - tbl, err := s.Table(ctx, qtid) + tbl, err := s.Table(tx, qtid) assert.NoError(t, err) assert.Equal(t, tableName, tbl.Name) } // Drop the table. - assert.NoError(t, s.DropTable(ctx, qtid)) + assert.NoError(t, s.DropTable(tx, qtid)) // Make sure the reverse-lookup (table by name) was dropped as well. { - _, err := s.TableID(ctx, qual, tableName) + _, err := s.TableID(tx, qdbid, tableName) if assert.Error(t, err) { assert.True(t, errors.Is(err, dax.ErrTableNameDoesNotExist)) } } // Try adding the table (i.e. the same table name) again. - assert.NoError(t, s.CreateTable(ctx, qtbl)) + assert.NoError(t, s.CreateTable(tx, qtbl)) // Drop the table again. - assert.NoError(t, s.DropTable(ctx, qtid)) + assert.NoError(t, s.DropTable(tx, qtid)) // Drop invalid table. { - iqtid := dax.NewQualifiedTableID(qual, invalidTableID) - err := s.DropTable(ctx, iqtid) + iqtid := dax.NewQualifiedTableID(qdbid, invalidTableID) + err := s.DropTable(tx, iqtid) if assert.Error(t, err) { assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist)) } @@ -99,19 +115,36 @@ func TestSchemar(t *testing.T) { }) t.Run("GetTables", func(t *testing.T) { + db := testbolt.MustOpenDB(t) + defer testbolt.MustCloseDB(t, db) + + t.Cleanup(func() { + testbolt.CleanupDB(t, db.Path()) + }) + + // Initialize the buckets. + assert.NoError(t, db.InitializeBuckets(boltdb.SchemarBuckets...)) + s := boltdb.NewSchemar(db, logger.NopLogger) + tx, err := db.BeginTx(ctx, true) + assert.NoError(t, err) + defer tx.Rollback() + + // Create database. + assert.NoError(t, s.CreateDatabase(tx, qdb)) + exp := []*dax.QualifiedTable{} - tables, err := s.Tables(ctx, qual) + tables, err := s.Tables(tx, qdbid) assert.NoError(t, err) assert.Equal(t, exp, tables) - qtbl0 := daxtest.TestQualifiedTableWithID(t, qual, tableID0, tableName0, partitionN, false) - qtbl1 := daxtest.TestQualifiedTableWithID(t, qual, tableID1, tableName1, partitionN, false) + qtbl0 := daxtest.TestQualifiedTableWithID(t, qdbid, tableID0, tableName0, partitionN, false) + qtbl1 := daxtest.TestQualifiedTableWithID(t, qdbid, tableID1, tableName1, partitionN, false) // Add a couple of tables. - assert.NoError(t, s.CreateTable(ctx, qtbl0)) - assert.NoError(t, s.CreateTable(ctx, qtbl1)) + assert.NoError(t, s.CreateTable(tx, qtbl0)) + assert.NoError(t, s.CreateTable(tx, qtbl1)) exp = []*dax.QualifiedTable{ qtbl1, @@ -119,62 +152,76 @@ func TestSchemar(t *testing.T) { } // All tables. - tables, err = s.Tables(ctx, qual) + tables, err = s.Tables(tx, qdbid) assert.NoError(t, err) assert.Equal(t, exp, tables) // With a valid filter. - tables, err = s.Tables(ctx, qual, qtbl0.ID) + tables, err = s.Tables(tx, qdbid, qtbl0.ID) assert.NoError(t, err) assert.Equal(t, exp[1:], tables) // With an invalid filter. - tables, err = s.Tables(ctx, qual, invalidTableID) + tables, err = s.Tables(tx, qdbid, invalidTableID) assert.NoError(t, err) assert.Equal(t, exp[0:0], tables) // With both valid and invalid filters. - tables, err = s.Tables(ctx, qual, qtbl0.ID, invalidTableID) + tables, err = s.Tables(tx, qdbid, qtbl0.ID, invalidTableID) assert.NoError(t, err) assert.Equal(t, exp[1:], tables) // With all valid filters. - tables, err = s.Tables(ctx, qual, qtbl0.ID, qtbl1.ID) + tables, err = s.Tables(tx, qdbid, qtbl0.ID, qtbl1.ID) assert.NoError(t, err) assert.Equal(t, exp, tables) }) t.Run("GetTablesAll", func(t *testing.T) { - // get a fresh DB db := testbolt.MustOpenDB(t) defer testbolt.MustCloseDB(t, db) t.Cleanup(func() { testbolt.CleanupDB(t, db.Path()) }) + // Initialize the buckets. assert.NoError(t, db.InitializeBuckets(boltdb.SchemarBuckets...)) s := boltdb.NewSchemar(db, logger.NopLogger) - qtbl0 := daxtest.TestQualifiedTableWithID(t, qual, tableID0, tableName0, partitionN, false) - orgID2 := dax.OrganizationID("acme2") - qual2 := dax.NewTableQualifier(orgID2, dbID) - tableID2 := "3" - qtbl2 := daxtest.TestQualifiedTableWithID(t, qual2, tableID2, dax.TableName("two"), partitionN, false) + tx, err := db.BeginTx(ctx, true) + assert.NoError(t, err) + defer tx.Rollback() - assert.NoError(t, s.CreateTable(ctx, qtbl0)) - assert.NoError(t, s.CreateTable(ctx, qtbl2)) + qtbl0 := daxtest.TestQualifiedTableWithID(t, qdbid, tableID0, tableName0, partitionN, false) + orgID2 := dax.OrganizationID("acme2") + qdbid2 := dax.NewQualifiedDatabaseID(orgID2, dbID) + tableID2 := "3" + qtbl2 := daxtest.TestQualifiedTableWithID(t, qdbid2, tableID2, dax.TableName("two"), partitionN, false) + + // Create databases. + assert.NoError(t, s.CreateDatabase(tx, qdb)) + qdb2 := &dax.QualifiedDatabase{ + OrganizationID: orgID2, + Database: dax.Database{ + ID: dbID, + Name: dbName, + }, + } + assert.NoError(t, s.CreateDatabase(tx, qdb2)) + + assert.NoError(t, s.CreateTable(tx, qtbl0)) + assert.NoError(t, s.CreateTable(tx, qtbl2)) exp := []*dax.QualifiedTable{qtbl0, qtbl2} - tables, err := s.Tables(ctx, dax.TableQualifier{}) + tables, err := s.Tables(tx, dax.QualifiedDatabaseID{}) assert.NoError(t, err) assert.Equal(t, exp, tables) - tables, err = s.Tables(ctx, dax.TableQualifier{OrganizationID: orgID2}) + tables, err = s.Tables(tx, dax.QualifiedDatabaseID{OrganizationID: orgID2}) assert.NoError(t, err) assert.Equal(t, []*dax.QualifiedTable{qtbl2}, tables) - }) } diff --git a/dax/mds/schemar/errors.go b/dax/mds/schemar/errors.go index 7e1025ec6..710ed9b4c 100644 --- a/dax/mds/schemar/errors.go +++ b/dax/mds/schemar/errors.go @@ -8,6 +8,9 @@ import ( ) const ( + ErrCodeDatabaseIDInvalid errors.Code = "DatabaseIDInvalid" + ErrCodeDatabaseNameInvalid errors.Code = "DatabaseNameInvalid" + ErrCodeTableIDInvalid errors.Code = "TableIDInvalid" ErrCodeTableNameInvalid errors.Code = "TableNameInvalid" ErrCodeInvalidPrimaryKey errors.Code = "InvalidPrimaryKey" @@ -15,6 +18,20 @@ const ( ErrCodeFieldNameInvalid errors.Code = "FieldNameInvalid" ) +func NewErrDatabaseIDInvalid(databaseID dax.DatabaseID) error { + return errors.New( + ErrCodeDatabaseIDInvalid, + fmt.Sprintf("database ID '%s' is invalid", databaseID), + ) +} + +func NewErrDatabaseNameInvalid(databaseName dax.DatabaseName) error { + return errors.New( + ErrCodeDatabaseNameInvalid, + fmt.Sprintf("database name '%s' is invalid", databaseName), + ) +} + func NewErrTableIDInvalid(tableID dax.TableID) error { return errors.New( ErrCodeTableIDInvalid, diff --git a/dax/mds/schemar/http/handler.go b/dax/mds/schemar/http/handler.go deleted file mode 100644 index eb8dd3f1f..000000000 --- a/dax/mds/schemar/http/handler.go +++ /dev/null @@ -1,199 +0,0 @@ -package http - -import ( - "encoding/json" - "net/http" - - "github.com/gorilla/mux" - "github.com/featurebasedb/featurebase/v3/dax" - "github.com/featurebasedb/featurebase/v3/dax/mds/schemar" -) - -func Handler(s schemar.Schemar) http.Handler { - svr := &server{ - schemar: s, - } - - router := mux.NewRouter() - router.HandleFunc("/health", svr.getHealth).Methods("GET").Name("GetHealth") - router.HandleFunc("/create-table", svr.postCreateTable).Methods("POST").Name("PostCreateTable") - router.HandleFunc("/drop-table", svr.postDropTable).Methods("POST").Name("PostDropTable") - router.HandleFunc("/table", svr.postTable).Methods("POST").Name("PostTable") - router.HandleFunc("/tables", svr.postTables).Methods("POST").Name("PostTables") - return router -} - -type server struct { - schemar schemar.Schemar -} - -// GET /health -func (s *server) getHealth(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) -} - -// POST /create-table -func (s *server) postCreateTable(w http.ResponseWriter, r *http.Request) { - body := r.Body - defer body.Close() - - ctx := r.Context() - - req := &dax.QualifiedTable{} - if err := json.NewDecoder(body).Decode(req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - err := s.schemar.CreateTable(ctx, req) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - resp := struct{}{} - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } -} - -// POST /drop-table -func (s *server) postDropTable(w http.ResponseWriter, r *http.Request) { - body := r.Body - defer body.Close() - - ctx := r.Context() - - req := DropTableRequest{} - if err := json.NewDecoder(body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - qtid := req.TableKey.QualifiedTableID() - - err := s.schemar.DropTable(ctx, qtid) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - resp := struct{}{} - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } -} - -// // POST /create-field -// func (s *server) postCreateField(w http.ResponseWriter, r *http.Request) { -// body := r.Body -// defer body.Close() - -// req := mds.CreateFieldRequest{} -// if err := json.NewDecoder(body).Decode(&req); err != nil { -// http.Error(w, err.Error(), http.StatusBadRequest) -// return -// } - -// resp, err := s.mds.CreateField(req) -// if err != nil { -// http.Error(w, err.Error(), http.StatusBadRequest) -// return -// } - -// if err := json.NewEncoder(w).Encode(resp); err != nil { -// http.Error(w, err.Error(), http.StatusBadRequest) -// return -// } -// } - -// // POST /drop-field -// func (s *server) postDropField(w http.ResponseWriter, r *http.Request) { -// body := r.Body -// defer body.Close() - -// req := mds.DropFieldRequest{} -// if err := json.NewDecoder(body).Decode(&req); err != nil { -// http.Error(w, err.Error(), http.StatusBadRequest) -// return -// } - -// resp, err := s.mds.DropField(req) -// if err != nil { -// http.Error(w, err.Error(), http.StatusBadRequest) -// return -// } - -// if err := json.NewEncoder(w).Encode(resp); err != nil { -// http.Error(w, err.Error(), http.StatusBadRequest) -// return -// } -// } - -// POST /table -func (s *server) postTable(w http.ResponseWriter, r *http.Request) { - body := r.Body - defer body.Close() - - ctx := r.Context() - - req := TableRequest{} - if err := json.NewDecoder(body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - qtid := req.TableKey.QualifiedTableID() - - resp, err := s.schemar.Table(ctx, qtid) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } -} - -// POST /tables -func (s *server) postTables(w http.ResponseWriter, r *http.Request) { - body := r.Body - defer body.Close() - - ctx := r.Context() - - req := TablesRequest{} - if err := json.NewDecoder(body).Decode(&req); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - qual := dax.NewTableQualifier(req.OrganizationID, req.DatabaseID) - resp, err := s.schemar.Tables(ctx, qual, req.TableIDs...) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if err := json.NewEncoder(w).Encode(resp); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } -} - -type DropTableRequest struct { - TableKey dax.TableKey `json:"table-key"` -} - -type TableRequest struct { - TableKey dax.TableKey `json:"table-key"` -} - -type TablesRequest struct { - OrganizationID dax.OrganizationID `json:"org-id"` - DatabaseID dax.DatabaseID `json:"db-id"` - TableIDs dax.TableIDs `json:"table-ids"` -} diff --git a/dax/mds/schemar/schemar.go b/dax/mds/schemar/schemar.go index 7f365268c..1b45dce9f 100644 --- a/dax/mds/schemar/schemar.go +++ b/dax/mds/schemar/schemar.go @@ -8,23 +8,34 @@ import ( ) type Schemar interface { - CreateTable(context.Context, *dax.QualifiedTable) error - DropTable(context.Context, dax.QualifiedTableID) error - CreateField(context.Context, dax.QualifiedTableID, *dax.Field) error - DropField(context.Context, dax.QualifiedTableID, dax.FieldName) error - Table(context.Context, dax.QualifiedTableID) (*dax.QualifiedTable, error) + CreateDatabase(dax.Transaction, *dax.QualifiedDatabase) error + DropDatabase(dax.Transaction, dax.QualifiedDatabaseID) error + DatabaseByID(dax.Transaction, dax.QualifiedDatabaseID) (*dax.QualifiedDatabase, error) - // Tables returns a list of tables. If the qualifiers DatabaseID - // is empty, all tables in the org will be returned. If the - // OrganizationID is empty, all tables will be returned. If both - // are populated, only tables in that databse will be returned. If - // greater than zero table IDs are passed in the third argument, - // only tables matching those IDs will be returned. - Tables(context.Context, dax.TableQualifier, ...dax.TableID) ([]*dax.QualifiedTable, error) + SetDatabaseOptions(dax.Transaction, dax.QualifiedDatabaseID, dax.DatabaseOptions) error + + // Databases returns a list of databases. If the OrganizationID is empty, + // all databases will be returned. If greater than zero database IDs are + // passed in the second argument, only databases matching those IDs will be + // returned. + Databases(dax.Transaction, dax.OrganizationID, ...dax.DatabaseID) ([]*dax.QualifiedDatabase, error) + + CreateTable(dax.Transaction, *dax.QualifiedTable) error + DropTable(dax.Transaction, dax.QualifiedTableID) error + CreateField(dax.Transaction, dax.QualifiedTableID, *dax.Field) error + DropField(dax.Transaction, dax.QualifiedTableID, dax.FieldName) error + Table(dax.Transaction, dax.QualifiedTableID) (*dax.QualifiedTable, error) + + // Tables returns a list of tables. If the qualifiers DatabaseID is empty, + // all tables in the org will be returned. If the OrganizationID is empty, + // all tables will be returned. If both are populated, only tables in that + // database will be returned. If greater than zero table IDs are passed in + // the third argument, only tables matching those IDs will be returned. + Tables(dax.Transaction, dax.QualifiedDatabaseID, ...dax.TableID) ([]*dax.QualifiedTable, error) // TableID is a reverse-lookup method to get the TableID for a given // qualified TableName. - TableID(context.Context, dax.TableQualifier, dax.TableName) (dax.QualifiedTableID, error) + TableID(dax.Transaction, dax.QualifiedDatabaseID, dax.TableName) (dax.QualifiedTableID, error) } ////////////////////////////////////////////// @@ -39,23 +50,50 @@ func NewNopSchemar() *NopSchemar { return &NopSchemar{} } -func (s *NopSchemar) CreateTable(ctx context.Context, qtbl *dax.QualifiedTable) error { return nil } -func (s *NopSchemar) DropTable(ctx context.Context, qtid dax.QualifiedTableID) error { +func (s *NopSchemar) CreateDatabase(tx dax.Transaction, qtbl *dax.QualifiedDatabase) error { return nil } -func (s *NopSchemar) CreateField(ctx context.Context, qtid dax.QualifiedTableID, fld *dax.Field) error { + +func (s *NopSchemar) DropDatabase(tx dax.Transaction, qdbid dax.QualifiedDatabaseID) error { return nil } -func (s *NopSchemar) DropField(ctx context.Context, qtid dax.QualifiedTableID, fld dax.FieldName) error { - return nil -} -func (s *NopSchemar) Table(ctx context.Context, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) { + +func (s *NopSchemar) DatabaseByID(dax.Transaction, dax.QualifiedDatabaseID) (*dax.QualifiedDatabase, error) { return nil, nil } -func (s *NopSchemar) Tables(ctx context.Context, qual dax.TableQualifier, ids ...dax.TableID) ([]*dax.QualifiedTable, error) { + +func (s *NopSchemar) SetDatabaseOptions(tx dax.Transaction, qdbid dax.QualifiedDatabaseID, opts dax.DatabaseOptions) error { + return nil +} + +func (s *NopSchemar) Databases(dax.Transaction, dax.OrganizationID, ...dax.DatabaseID) ([]*dax.QualifiedDatabase, error) { + return nil, nil +} + +func (s *NopSchemar) CreateTable(tx dax.Transaction, qtbl *dax.QualifiedTable) error { + return nil +} + +func (s *NopSchemar) DropTable(tx dax.Transaction, qtid dax.QualifiedTableID) error { + return nil +} + +func (s *NopSchemar) CreateField(tx dax.Transaction, qtid dax.QualifiedTableID, fld *dax.Field) error { + return nil +} + +func (s *NopSchemar) DropField(tx dax.Transaction, qtid dax.QualifiedTableID, fld dax.FieldName) error { + return nil +} + +func (s *NopSchemar) Table(tx dax.Transaction, qtid dax.QualifiedTableID) (*dax.QualifiedTable, error) { + return nil, nil +} + +func (s *NopSchemar) Tables(tx dax.Transaction, qdbid dax.QualifiedDatabaseID, ids ...dax.TableID) ([]*dax.QualifiedTable, error) { return []*dax.QualifiedTable{}, nil } -func (s *NopSchemar) TableID(context.Context, dax.TableQualifier, dax.TableName) (dax.QualifiedTableID, error) { +func (s *NopSchemar) TableID(dax.Transaction, dax.QualifiedDatabaseID, dax.TableName) (dax.QualifiedTableID, error) { return dax.QualifiedTableID{}, nil } diff --git a/dax/mds/schemar/schemar_test.go b/dax/mds/schemar/schemar_test.go deleted file mode 100644 index 8f0904af3..000000000 --- a/dax/mds/schemar/schemar_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package schemar_test - -import ( - "context" - "testing" - - "github.com/featurebasedb/featurebase/v3/dax" - daxtest "github.com/featurebasedb/featurebase/v3/dax/test" - "github.com/featurebasedb/featurebase/v3/errors" - "github.com/stretchr/testify/assert" -) - -func TestSchemar(t *testing.T) { - orgID := dax.OrganizationID("acme") - dbID := dax.DatabaseID("db1") - invalidTableID := dax.TableID("invalidID") - tableName := dax.TableName("foo") - tableName0 := dax.TableName("foo") - tableName1 := dax.TableName("bar") - tableID0 := "2" - tableID1 := "1" - partitionN := 12 - - ctx := context.Background() - qual := dax.NewTableQualifier(orgID, dbID) - - t.Run("NewSchemar", func(t *testing.T) { - s, cleanup := daxtest.NewSchemar(t) - defer cleanup() - - // Add new table. - tbl := dax.NewTable(tableName) - tbl.Fields = []*dax.Field{ - { - Name: dax.PrimaryKeyFieldName, - Type: dax.BaseTypeString, - }, - { - Name: "intField", - Type: dax.BaseTypeInt, - }, - } - qtbl := dax.NewQualifiedTable(qual, tbl) - qtbl.CreateID() - assert.NoError(t, s.CreateTable(ctx, qtbl)) - - // Try adding the table again. - err := s.CreateTable(ctx, qtbl) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrTableIDExists)) - } - - qtid := qtbl.QualifiedID() - - // Get the table. - { - tbl, err := s.Table(ctx, qtid) - assert.NoError(t, err) - assert.Equal(t, tableName, tbl.Name) - } - - // Drop the table. - { - err := s.DropTable(ctx, qtid) - assert.NoError(t, err) - } - - // Drop invalid table. - { - iqtid := dax.NewQualifiedTableID(qual, invalidTableID) - err := s.DropTable(ctx, iqtid) - if assert.Error(t, err) { - assert.True(t, errors.Is(err, dax.ErrTableIDDoesNotExist)) - } - } - }) - - t.Run("GetTables", func(t *testing.T) { - s, cleanup := daxtest.NewSchemar(t) - defer cleanup() - - exp := []*dax.QualifiedTable{} - tables, err := s.Tables(ctx, qual) - assert.NoError(t, err) - assert.Equal(t, exp, tables) - - qtbl0 := daxtest.TestQualifiedTableWithID(t, qual, tableID0, tableName0, partitionN, false) - qtbl1 := daxtest.TestQualifiedTableWithID(t, qual, tableID1, tableName1, partitionN, false) - - // Add a couple of tables. - assert.NoError(t, s.CreateTable(ctx, qtbl0)) - assert.NoError(t, s.CreateTable(ctx, qtbl1)) - - exp = []*dax.QualifiedTable{ - qtbl1, - qtbl0, - } - - // All tables. - tables, err = s.Tables(ctx, qual) - assert.NoError(t, err) - assert.Equal(t, exp, tables) - - // With a valid filter. - tables, err = s.Tables(ctx, qual, qtbl0.ID) - assert.NoError(t, err) - assert.Equal(t, exp[1:], tables) - - // With an invalid filter. - tables, err = s.Tables(ctx, qual, invalidTableID) - assert.NoError(t, err) - assert.Equal(t, exp[0:0], tables) - - // With both valid and invalid filters. - tables, err = s.Tables(ctx, qual, qtbl0.ID, invalidTableID) - assert.NoError(t, err) - assert.Equal(t, exp[1:], tables) - - // With all valid filters. - tables, err = s.Tables(ctx, qual, qtbl0.ID, qtbl1.ID) - assert.NoError(t, err) - assert.Equal(t, exp, tables) - }) -} diff --git a/dax/node.go b/dax/node.go index bd130ee95..d1ddb3a03 100644 --- a/dax/node.go +++ b/dax/node.go @@ -3,6 +3,7 @@ package dax import ( "context" "fmt" + "strings" "github.com/featurebasedb/featurebase/v3/errors" ) @@ -15,7 +16,23 @@ type Node struct { RoleTypes []RoleType `json:"role-types"` } -// AssignedNode is used in API responses. +// Nodes is a slice of *Node. It's useful for printing the nodes as a list of +// node.Addresses via its String() method. +type Nodes []*Node + +// String prints the slice of node addresses in Nodes. +func (n Nodes) String() string { + out := make([]string, 0, len(n)) + for i := range n { + out = append(out, string(n[i].Address)) + } + return "[" + strings.Join(out, ",") + "]" +} + +// AssignedNode represents a Worker which has been assigned a role. Note that +// the worker which it represents might be responsible for multiple roles, but +// AssignedNode only ever represents one of those roles at a time. This is +// because it is always the response of a RoleType-specific request. type AssignedNode struct { Address Address `json:"address"` Role Role `json:"role"` @@ -23,10 +40,10 @@ type AssignedNode struct { // NodeService represents a service for managing Nodes. type NodeService interface { - CreateNode(context.Context, Address, *Node) error - ReadNode(context.Context, Address) (*Node, error) - DeleteNode(context.Context, Address) error - Nodes(context.Context) ([]*Node, error) + CreateNode(Transaction, Address, *Node) error + ReadNode(Transaction, Address) (*Node, error) + DeleteNode(Transaction, Address) error + Nodes(Transaction) ([]*Node, error) } // ComputeNode represents a compute node and the table/shards for which it is diff --git a/dax/queryer/client/client.go b/dax/queryer/client/client.go index 6e438c4d8..eead12ca8 100644 --- a/dax/queryer/client/client.go +++ b/dax/queryer/client/client.go @@ -49,12 +49,12 @@ func (c *Client) Health() bool { return true } -func (c *Client) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql string) (*featurebase.WireQueryResponse, error) { +func (c *Client) QuerySQL(ctx context.Context, qdbid dax.QualifiedDatabaseID, sql string) (*featurebase.WireQueryResponse, error) { url := fmt.Sprintf("%s/sql", c.address.WithScheme(defaultScheme)) req := &queryerhttp.SQLRequest{ - OrganizationID: qual.OrganizationID, - DatabaseID: qual.DatabaseID, + OrganizationID: qdbid.OrganizationID, + DatabaseID: qdbid.DatabaseID, SQL: sql, } @@ -85,12 +85,13 @@ func (c *Client) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql stri return wireResp, nil } -func (c *Client) QueryPQL(ctx context.Context, qual dax.TableQualifier, table dax.TableName, pql string) (*featurebase.WireQueryResponse, error) { + +func (c *Client) QueryPQL(ctx context.Context, qdbid dax.QualifiedDatabaseID, table dax.TableName, pql string) (*featurebase.WireQueryResponse, error) { url := fmt.Sprintf("%s/query", c.address.WithScheme(defaultScheme)) req := &queryerhttp.QueryRequest{ - OrganizationID: qual.OrganizationID, - DatabaseID: qual.DatabaseID, + OrganizationID: qdbid.OrganizationID, + DatabaseID: qdbid.DatabaseID, Table: table, PQL: pql, } diff --git a/dax/queryer/http/handler.go b/dax/queryer/http/handler.go index de344086b..419336bd3 100644 --- a/dax/queryer/http/handler.go +++ b/dax/queryer/http/handler.go @@ -49,15 +49,15 @@ func (s *server) postQuery(w http.ResponseWriter, r *http.Request) { var resp interface{} var err error - qual := dax.NewTableQualifier(req.OrganizationID, req.DatabaseID) + qdbid := dax.NewQualifiedDatabaseID(req.OrganizationID, req.DatabaseID) if req.SQL != "" { - resp, err = s.queryer.QuerySQL(ctx, qual, req.SQL) + resp, err = s.queryer.QuerySQL(ctx, qdbid, req.SQL) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } } else { - resp, err = s.queryer.QueryPQL(ctx, qual, req.Table, req.PQL) + resp, err = s.queryer.QueryPQL(ctx, qdbid, req.Table, req.PQL) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -83,8 +83,8 @@ func (s *server) postSQL(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - qual := dax.NewTableQualifier(req.OrganizationID, req.DatabaseID) - resp, err := s.queryer.QuerySQL(ctx, qual, req.SQL) + qdbid := dax.NewQualifiedDatabaseID(req.OrganizationID, req.DatabaseID) + resp, err := s.queryer.QuerySQL(ctx, qdbid, req.SQL) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return diff --git a/dax/queryer/orchestrator.go b/dax/queryer/orchestrator.go index 15b28e16e..5d6ce48c9 100644 --- a/dax/queryer/orchestrator.go +++ b/dax/queryer/orchestrator.go @@ -1840,6 +1840,7 @@ func (o *orchestrator) executeExtract(ctx context.Context, tableKeyer dax.TableK } fields[i] = fieldName } + // TODO(tlt): is `fields` used? // Merge returned results at coordinating node. reduceFn := func(ctx context.Context, prev, v interface{}) interface{} { @@ -3511,13 +3512,13 @@ func callArgString(call *pql.Call, key string) string { type qualifiedOrchestrator struct { *orchestrator - qual dax.TableQualifier + qdbid dax.QualifiedDatabaseID } -func newQualifiedOrchestrator(orch *orchestrator, qual dax.TableQualifier) *qualifiedOrchestrator { +func newQualifiedOrchestrator(orch *orchestrator, qdbid dax.QualifiedDatabaseID) *qualifiedOrchestrator { return &qualifiedOrchestrator{ orchestrator: orch, - qual: qual, + qdbid: qdbid, } } @@ -3528,7 +3529,7 @@ func (o *qualifiedOrchestrator) Execute(ctx context.Context, tableKeyer dax.Tabl switch keyer := tableKeyer.(type) { case *dax.Table: - qtbl = dax.NewQualifiedTable(o.qual, keyer) + qtbl = dax.NewQualifiedTable(o.qdbid, keyer) case *dax.QualifiedTable: qtbl = keyer default: diff --git a/dax/queryer/queryer.go b/dax/queryer/queryer.go index 1356e8e8d..8b052d2db 100644 --- a/dax/queryer/queryer.go +++ b/dax/queryer/queryer.go @@ -32,7 +32,7 @@ import ( // of "Queryer" nodes, which handle incoming query requests. type Queryer struct { mu sync.RWMutex - orchestrators map[dax.TableQualifier]*qualifiedOrchestrator + orchestrators map[dax.QualifiedDatabaseID]*qualifiedOrchestrator fbClient *featurebase.InternalClient @@ -47,7 +47,7 @@ func New(cfg Config) *Queryer { q := &Queryer{ noder: dax.NewNopNoder(), schemar: dax.NewNopSchemar(), - orchestrators: make(map[dax.TableQualifier]*qualifiedOrchestrator), + orchestrators: make(map[dax.QualifiedDatabaseID]*qualifiedOrchestrator), logger: logger.NopLogger, } @@ -59,13 +59,13 @@ func New(cfg Config) *Queryer { } // Orchestrator gets (or creates) an instance of qualifiedOrchestrator based on -// the provided dax.TableQualifier. -func (q *Queryer) Orchestrator(qual dax.TableQualifier) *qualifiedOrchestrator { +// the provided dax.QualifiedDatabaseID. +func (q *Queryer) Orchestrator(qdbid dax.QualifiedDatabaseID) *qualifiedOrchestrator { // Try to get orchestrator under a read lock first. if orch := func() *qualifiedOrchestrator { q.mu.RLock() defer q.mu.RUnlock() - if orch, ok := q.orchestrators[qual]; ok { + if orch, ok := q.orchestrators[qdbid]; ok { return orch } return nil @@ -77,11 +77,11 @@ func (q *Queryer) Orchestrator(qual dax.TableQualifier) *qualifiedOrchestrator { // lock and try a read/write. q.mu.Lock() defer q.mu.Unlock() - if orch, ok := q.orchestrators[qual]; ok { + if orch, ok := q.orchestrators[qdbid]; ok { return orch } - sapi := newQualifiedSchemaAPI(qual, q.schemar) + sapi := newQualifiedSchemaAPI(qdbid, q.schemar) orch := &orchestrator{ schema: sapi, @@ -92,8 +92,8 @@ func (q *Queryer) Orchestrator(qual dax.TableQualifier) *qualifiedOrchestrator { logger: q.logger, } - qorch := newQualifiedOrchestrator(orch, qual) - q.orchestrators[qual] = qorch + qorch := newQualifiedOrchestrator(orch, qdbid) + q.orchestrators[qdbid] = qorch return qorch } @@ -132,12 +132,9 @@ func (q *Queryer) Start() error { return nil } -func (q *Queryer) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql string) (*featurebase.WireQueryResponse, error) { +func (q *Queryer) QuerySQL(ctx context.Context, qdbid dax.QualifiedDatabaseID, sql string) (*featurebase.WireQueryResponse, error) { start := time.Now() - if len(sql) > 0 && sql[0] == '[' { - return q.parseAndQueryPQL(ctx, qual, sql) - } ret := &featurebase.WireQueryResponse{} applyExecutionTime := func() { @@ -149,6 +146,19 @@ func (q *Queryer) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql str applyExecutionTime() } + // If PQL, run that instead. + if len(sql) > 0 && sql[0] == '[' { + if pqlResp, err := q.parseAndQueryPQL(ctx, qdbid, sql); err != nil { + applyError(errors.Wrap(err, "querying pql")) + return ret, nil + } else { + ret = pqlResp + } + applyExecutionTime() + + return ret, nil + } + // Create a requestID and add it to the context. requestID, err := uuid.NewV4() if err != nil { @@ -165,19 +175,17 @@ func (q *Queryer) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql str } // SchemaAPI - sapi := newQualifiedSchemaAPI(qual, q.schemar) + sapi := newQualifiedSchemaAPI(qdbid, q.schemar) // Importer - imp := idkmds.NewImporter(q.noder, q.schemar, qual, nil) + imp := idkmds.NewImporter(q.noder, q.schemar, qdbid, nil) - // TODO(tlt): this obviously doesn't work; we don't have an API here. We - // need a dax-compatible implementation of the SystemAPI (or at least a - // no-op implementation). - sysapi := &featurebase.FeatureBaseSystemAPI{API: nil} + // TODO(tlt): We need a dax-compatible implementation of the SystemAPI. + sysapi := &featurebase.NopSystemAPI{} systemLayer := systemlayer.NewSystemLayer() - pl := planner.NewExecutionPlanner(q.Orchestrator(qual), sapi, sysapi, systemLayer, imp, q.logger, sql) + pl := planner.NewExecutionPlanner(q.Orchestrator(qdbid), sapi, sysapi, systemLayer, imp, q.logger, sql) planOp, err := pl.CompilePlan(ctx, st) if err != nil { @@ -229,7 +237,7 @@ func (q *Queryer) QuerySQL(ctx context.Context, qual dax.TableQualifier, sql str return ret, nil } -func (q *Queryer) parseAndQueryPQL(ctx context.Context, qual dax.TableQualifier, sql string) (*featurebase.WireQueryResponse, error) { +func (q *Queryer) parseAndQueryPQL(ctx context.Context, qdbid dax.QualifiedDatabaseID, sql string) (*featurebase.WireQueryResponse, error) { var i int for i = 1; sql[i] != ']'; i++ { if i == len(sql)-1 { @@ -240,7 +248,7 @@ func (q *Queryer) parseAndQueryPQL(ctx context.Context, qual dax.TableQualifier, query := sql[i+1:] fmt.Println("got table/query", table, query) - return q.QueryPQL(ctx, qual, dax.TableName(table), query) + return q.queryPQL(ctx, qdbid, dax.TableName(table), query) } // convertIndex tries to covert any "index" specified in the call.Args map to a @@ -250,9 +258,9 @@ func (q *Queryer) parseAndQueryPQL(ctx context.Context, qual dax.TableQualifier, // modify Call.CallIndex() to be TableKeyer aware. I didn't do that along with // these changes because I'm not sure if we want to introduce dax types into the // pql package. -func (q *Queryer) convertIndex(ctx context.Context, qual dax.TableQualifier, call *featurebase_pql.Call) { +func (q *Queryer) convertIndex(ctx context.Context, qdbid dax.QualifiedDatabaseID, call *featurebase_pql.Call) { if index := call.CallIndex(); index != "" { - qtbl, err := q.schemar.TableByName(ctx, qual, dax.TableName(index)) + qtbl, err := q.schemar.TableByName(ctx, qdbid, dax.TableName(index)) if err != nil { return } @@ -261,11 +269,37 @@ func (q *Queryer) convertIndex(ctx context.Context, qual dax.TableQualifier, cal // Apply to children. for _, child := range call.Children { - q.convertIndex(ctx, qual, child) + q.convertIndex(ctx, qdbid, child) } } -func (q *Queryer) QueryPQL(ctx context.Context, qual dax.TableQualifier, table dax.TableName, pql string) (*featurebase.WireQueryResponse, error) { +func (q *Queryer) QueryPQL(ctx context.Context, qdbid dax.QualifiedDatabaseID, table dax.TableName, pql string) (*featurebase.WireQueryResponse, error) { + start := time.Now() + + ret := &featurebase.WireQueryResponse{} + + applyExecutionTime := func() { + ret.ExecutionTime = time.Since(start).Microseconds() + } + + applyError := func(e error) { + ret.Error = e.Error() + applyExecutionTime() + } + + if pqlResp, err := q.queryPQL(ctx, qdbid, table, pql); err != nil { + applyError(errors.Wrap(err, "querying pql")) + return ret, nil + } else { + ret = pqlResp + } + + applyExecutionTime() + + return ret, nil +} + +func (q *Queryer) queryPQL(ctx context.Context, qdbid dax.QualifiedDatabaseID, table dax.TableName, pql string) (*featurebase.WireQueryResponse, error) { // Parse the pql into a pql.Query containing []pql.Call. qry, err := featurebase_pql.NewParser(strings.NewReader(pql)).Parse() if err != nil { @@ -276,14 +310,14 @@ func (q *Queryer) QueryPQL(ctx context.Context, qual dax.TableQualifier, table d } // Replace any "index" arguments within the PQL with a TableKey. - q.convertIndex(ctx, qual, qry.Calls[0]) + q.convertIndex(ctx, qdbid, qry.Calls[0]) - qtbl, err := q.schemar.TableByName(ctx, qual, dax.TableName(table)) + qtbl, err := q.schemar.TableByName(ctx, qdbid, dax.TableName(table)) if err != nil { return nil, errors.Wrap(err, "converting index to qualified table") } - results, err := q.Orchestrator(qual).Execute(ctx, qtbl, qry, nil, &featurebase.ExecOptions{}) + results, err := q.Orchestrator(qdbid).Execute(ctx, qtbl, qry, nil, &featurebase.ExecOptions{}) if err != nil { return nil, errors.Wrap(err, "orchestrator.Execute") } diff --git a/dax/queryer/schema_api.go b/dax/queryer/schema_api.go index 19405f832..52c638ec0 100644 --- a/dax/queryer/schema_api.go +++ b/dax/queryer/schema_api.go @@ -12,31 +12,31 @@ import ( var _ pilosa.SchemaAPI = (*qualifiedSchemaAPI)(nil) // qualifiedSchemaAPI is a wrapper around schemaAPI. It is initialized with a -// TableQualifier, and it uses this qualifer to convert between, for example, -// FeatureBase index name (a string) and TableKey. It requires a Schemar to do -// that lookup/conversion. +// QualifiedDatabaseID, and it uses this qualifer to convert between, for +// example, FeatureBase index name (a string) and TableKey. It requires a +// Schemar to do that lookup/conversion. type qualifiedSchemaAPI struct { - qual dax.TableQualifier + qdbid dax.QualifiedDatabaseID schemar dax.Schemar } -func newQualifiedSchemaAPI(qual dax.TableQualifier, schema dax.Schemar) *qualifiedSchemaAPI { +func newQualifiedSchemaAPI(qdbid dax.QualifiedDatabaseID, schema dax.Schemar) *qualifiedSchemaAPI { return &qualifiedSchemaAPI{ - qual: qual, + qdbid: qdbid, schemar: schema, } } func (s *qualifiedSchemaAPI) TableByName(ctx context.Context, tname dax.TableName) (*dax.Table, error) { - qtbl, err := s.schemar.TableByName(ctx, s.qual, tname) + qtbl, err := s.schemar.TableByName(ctx, s.qdbid, tname) if err != nil { - return nil, errors.Wrapf(err, "getting table id: (%s) %s", s.qual, tname) + return nil, errors.Wrapf(err, "getting table id: (%s) %s", s.qdbid, tname) } return &qtbl.Table, nil } func (s *qualifiedSchemaAPI) TableByID(ctx context.Context, tid dax.TableID) (*dax.Table, error) { - qtid := dax.NewQualifiedTableID(s.qual, tid) + qtid := dax.NewQualifiedTableID(s.qdbid, tid) qtbl, err := s.schemar.TableByID(ctx, qtid) if err != nil { @@ -47,7 +47,7 @@ func (s *qualifiedSchemaAPI) TableByID(ctx context.Context, tid dax.TableID) (*d } func (s *qualifiedSchemaAPI) Tables(ctx context.Context) ([]*dax.Table, error) { - qtbls, err := s.schemar.Tables(ctx, s.qual) + qtbls, err := s.schemar.Tables(ctx, s.qdbid) if err != nil { return nil, errors.Wrap(err, "getting tables") } @@ -61,32 +61,32 @@ func (s *qualifiedSchemaAPI) Tables(ctx context.Context) ([]*dax.Table, error) { } func (s *qualifiedSchemaAPI) CreateTable(ctx context.Context, tbl *dax.Table) error { - qtbl := dax.NewQualifiedTable(s.qual, tbl) + qtbl := dax.NewQualifiedTable(s.qdbid, tbl) return s.schemar.CreateTable(ctx, qtbl) } func (s *qualifiedSchemaAPI) CreateField(ctx context.Context, tname dax.TableName, fld *dax.Field) error { - qtbl, err := s.schemar.TableByName(ctx, s.qual, tname) + qtbl, err := s.schemar.TableByName(ctx, s.qdbid, tname) if err != nil { - return errors.Wrapf(err, "getting table by name: (%s) %s", s.qual, tname) + return errors.Wrapf(err, "getting table by name: (%s) %s", s.qdbid, tname) } return s.schemar.CreateField(ctx, qtbl.QualifiedID(), fld) } func (s *qualifiedSchemaAPI) DeleteTable(ctx context.Context, tname dax.TableName) error { - qtbl, err := s.schemar.TableByName(ctx, s.qual, tname) + qtbl, err := s.schemar.TableByName(ctx, s.qdbid, tname) if err != nil { - return errors.Wrapf(err, "getting table by name: (%s) %s", s.qual, tname) + return errors.Wrapf(err, "getting table by name: (%s) %s", s.qdbid, tname) } return s.schemar.DropTable(ctx, qtbl.QualifiedID()) } func (s *qualifiedSchemaAPI) DeleteField(ctx context.Context, tname dax.TableName, fname dax.FieldName) error { - qtid, err := s.schemar.TableByName(ctx, s.qual, tname) + qtid, err := s.schemar.TableByName(ctx, s.qdbid, tname) if err != nil { - return errors.Wrapf(err, "getting table by name: (%s) %s", s.qual, tname) + return errors.Wrapf(err, "getting table by name: (%s) %s", s.qdbid, tname) } return s.schemar.DropField(ctx, qtid.Key().QualifiedTableID(), fname) diff --git a/dax/role.go b/dax/role.go index 92712ca64..165356611 100644 --- a/dax/role.go +++ b/dax/role.go @@ -28,8 +28,8 @@ type Role interface { } // Ensure type implements interface. -var _ Role = &ComputeRole{} -var _ Role = &TranslateRole{} +var _ Role = (*ComputeRole)(nil) +var _ Role = (*TranslateRole)(nil) // ComputeRole is a role specific to compute nodes. type ComputeRole struct { @@ -37,7 +37,7 @@ type ComputeRole struct { Shards ShardNums `json:"shards"` } -// Type returns the type for ComputeRole. This is mainly to impolement the Role +// Type returns the type for ComputeRole. This is mainly to implement the Role // interface. func (cr *ComputeRole) Type() RoleType { return RoleTypeCompute @@ -50,7 +50,7 @@ type TranslateRole struct { Fields []FieldName `json:"fields"` } -// Type returns the type for TransteRole. This is mainly to impolement the Role +// Type returns the type for TranslateRole. This is mainly to implement the Role // interface. func (cr *TranslateRole) Type() RoleType { return RoleTypeTranslate diff --git a/dax/schema.go b/dax/schema.go index f7b16abc4..05d31c43e 100644 --- a/dax/schema.go +++ b/dax/schema.go @@ -3,16 +3,52 @@ package dax import "context" // Schemar is similar to the pilosa.SchemaAPI interface, but it takes -// TableQualifiers into account. +// QualifiedDatabaseIDs into account. Note that it is also similar to the +// schemar.Schemar interface, but that is used internally, typically within the +// Controller, and it takes Transactions rather than a Context, because its +// methods are assumed to be used as part of a larger request. +// TODO(tlt): clean up the mds/controller/schemar Schemar interface confusion. type Schemar interface { - TableByName(ctx context.Context, qual TableQualifier, tname TableName) (*QualifiedTable, error) - TableByID(ctx context.Context, qtid QualifiedTableID) (*QualifiedTable, error) - Tables(ctx context.Context, qual TableQualifier, tids ...TableID) ([]*QualifiedTable, error) + ////////////////////////////////////////////////////////////////////////// + // Database methods + ////////////////////////////////////////////////////////////////////////// + + CreateDatabase(context.Context, *QualifiedDatabase) error + // DropDatabase(context.Context, QualifiedDatabaseID) error + + // DatabaseByName(ctx context.Context, orgID OrganizationID, dbname DatabaseName) (*QualifiedDatabase, error) + DatabaseByID(ctx context.Context, qdbid QualifiedDatabaseID) (*QualifiedDatabase, error) + + // // Databases returns a list of databases. If the OrganizationID is empty, + // // all databases will be returned. If greater than zero database IDs are + // // passed in the second argument, only databases matching those IDs will be + // // returned. + // Databases(context.Context, OrganizationID, ...DatabaseID) ([]*QualifiedDatabase, error) + + // SetDatabaseOptions(context.Context, QualifiedDatabaseID, DatabaseOptions) error + + ////////////////////////////////////////////////////////////////////////// + // Table methods + ////////////////////////////////////////////////////////////////////////// CreateTable(ctx context.Context, qtbl *QualifiedTable) error - CreateField(ctx context.Context, qtid QualifiedTableID, fld *Field) error - DropTable(ctx context.Context, qtid QualifiedTableID) error + + TableByName(ctx context.Context, qdbid QualifiedDatabaseID, tname TableName) (*QualifiedTable, error) + TableByID(ctx context.Context, qtid QualifiedTableID) (*QualifiedTable, error) + + // Tables returns a list of tables. If the qualifiers DatabaseID is empty, + // all tables in the org will be returned. If the OrganizationID is empty, + // all tables will be returned. If both are populated, only tables in that + // database will be returned. If greater than zero table IDs are passed in + // the third argument, only tables matching those IDs will be returned. + Tables(ctx context.Context, qdbid QualifiedDatabaseID, tids ...TableID) ([]*QualifiedTable, error) + + ////////////////////////////////////////////////////////////////////////// + // Field methods + ////////////////////////////////////////////////////////////////////////// + + CreateField(ctx context.Context, qtid QualifiedTableID, fld *Field) error DropField(ctx context.Context, qtid QualifiedTableID, fname FieldName) error } @@ -28,13 +64,19 @@ func NewNopSchemar() *NopSchemar { return &NopSchemar{} } -func (s *NopSchemar) TableByName(context.Context, TableQualifier, TableName) (*QualifiedTable, error) { +func (s *NopSchemar) CreateDatabase(context.Context, *QualifiedDatabase) error { + return nil +} +func (s *NopSchemar) DatabaseByID(ctx context.Context, qdbid QualifiedDatabaseID) (*QualifiedDatabase, error) { + return nil, nil +} +func (s *NopSchemar) TableByName(context.Context, QualifiedDatabaseID, TableName) (*QualifiedTable, error) { return nil, nil } func (s *NopSchemar) TableByID(ctx context.Context, qtid QualifiedTableID) (*QualifiedTable, error) { return nil, nil } -func (s *NopSchemar) Tables(ctx context.Context, qual TableQualifier, tids ...TableID) ([]*QualifiedTable, error) { +func (s *NopSchemar) Tables(ctx context.Context, qdbid QualifiedDatabaseID, tids ...TableID) ([]*QualifiedTable, error) { return nil, nil } func (s *NopSchemar) CreateTable(ctx context.Context, qtbl *QualifiedTable) error { diff --git a/dax/storage/storage_test.go b/dax/storage/storage_test.go index 0b368569a..081133580 100644 --- a/dax/storage/storage_test.go +++ b/dax/storage/storage_test.go @@ -34,10 +34,10 @@ func TestResourceManager(t *testing.T) { mm := NewResourceManager(sn, wl, logger.NewStandardLogger(os.Stderr)) qtid := dax.QualifiedTableID{ - TableQualifier: dax.TableQualifier{ - OrganizationID: dax.OrganizationID("org1"), - DatabaseID: dax.DatabaseID("db1"), - }, + QualifiedDatabaseID: dax.NewQualifiedDatabaseID( + dax.OrganizationID("org1"), + dax.DatabaseID("db1"), + ), ID: dax.TableID("blah"), Name: "blah", } diff --git a/dax/table.go b/dax/table.go index eb39df22a..577174872 100644 --- a/dax/table.go +++ b/dax/table.go @@ -22,14 +22,17 @@ import ( // // OrganizationID - carried over from ControlPlane; currently uuid // DatabaseID - carried over from ControlPlane; currently uuid +// Database - base Database struct +// DatabaseKey - a string representation of OrganizationID and DatabaseID // TableID - internally stored as a uint64; presented as a hex string. // TableName - human-friendly string table name // Table - base Table struct; includes a TableID and a TableName -// TableQualifier - combination of OrganizationID and DatabaseID -// QualifiedTable - TableQualifier plus a Table -// QualifiedTableID - TableQualifier plus a TableID +// QualifiedDatabase - OrganizationID plus a Database +// QualifiedDatabaseID - combination of OrganizationID and DatabaseID +// QualifiedTable - QualifiedDatabaseID plus a Table +// QualifiedTableID - QualifiedDatabaseID plus a TableID // TableKey - a string representation of OrganizationID, DatabaseID, and -// TableID, which is safe to use as a FeatureBase index name. +// TableID, which is safe to use as a FeatureBase index name. // // Example: // OrganizationID - "29-ae44-41" @@ -37,7 +40,7 @@ import ( // TableID - 123456789 (hex string: "499602d2") // TableName - foo // Table - {ID:"499602d2", Name: "foo", Fields: ... } -// TableQualifier - {Org: "29-ae44-41", DB: "75-d1a2-4f"} +// QualifierDatabaseID - {Org: "29-ae44-41", DB: "75-d1a2-4f"} // QualifiedTable - {Org: "29-ae44-41", DB: "75-d1a2-4f", Table: *tbl} // QualifiedTableID - {Org: "29-ae44-41", DB: "75-d1a2-4f", TableID: "499602d2"} // TableKey - "tbl__29-ae44-41__75-d1a2-4f__499602d2" @@ -52,6 +55,13 @@ import ( // underscore. const TableKeyDelimiter = "__" +// PrefixDatabase is used as a prefix to DatabaseKey strings because FeatureBase +// indexes must start with an alpha (a-z) character. Because the string +// representation of a uuid (i.e. the OrganizationID value) can start with a +// numeric value, we can't have OrganizationId (or any of the other ID values +// which make up the DatabaseKey) be at the beginning of the DatabaseKey. +const PrefixDatabase = "db" + // PrefixTable is used as a prefix to TableKey strings because FeatureBase // indexes must start with an alpha (a-z) character. Because the string // representation of a uuid (i.e. the OrganizationID value) can start with a @@ -103,6 +113,124 @@ type OrganizationID string // value could be any string. type DatabaseID string +// DatabaseKey is a globally unique identifier for a database; it is effectively the +// compound key: (org, database). This is (hopefully) the value that will +// be used when interfacing with services which are unaware of qualifiers. +type DatabaseKey string + +// QualifiedDatabaseID returns the QualifiedDatabaseID based on the key. If +// DatabaseKey can't be parsed into a valid (i.e. complete) QualifiedDatabaseID, +// then blank values are used where necessary. +func (dk DatabaseKey) QualifiedDatabaseID() QualifiedDatabaseID { + qdbid, err := QualifiedDatabaseIDFromKey(string(dk)) + if err != nil { + return NewQualifiedDatabaseID("", DatabaseID(dk)) + } + return qdbid +} + +// DatabaseName is a human-friendly string. +type DatabaseName string + +// Database represents a database and its configuration. +type Database struct { + ID DatabaseID `json:"id"` + Name DatabaseName `json:"name"` + Options DatabaseOptions `json:"options"` + // Tables []*Table `json:"tables"` + + Description string `json:"description,omitempty"` + Owner string `json:"owner,omitempty"` + CreatedAt int64 `json:"createdAt,omitempty"` + UpdatedAt int64 `json:"updatedAt,omitempty"` + UpdatedBy string `json:"updatedBy,omitempty"` +} + +// DatabaseOptions are used to configure a database. +type DatabaseOptions struct { + WorkersMin int `json:"workers-min"` + WorkersMax int `json:"workers-max"` +} + +// QualifiedDatabase is a Database along with its OrganizationID. +type QualifiedDatabase struct { + OrganizationID OrganizationID `json:"org-id"` + Database +} + +type QualifiedDatabases []*QualifiedDatabase + +// Key returns the string-encoded (delimited by DatabaseKeyDelimiter) globally +// unique DatabaseKey. +func (qdb QualifiedDatabase) Key() DatabaseKey { + return qdb.QualifiedID().Key() +} + +// String returns a human-friendly version of the QualifiedDatabase. It is only +// used for display purposes; it is not used as any kind of key. +func (qdb QualifiedDatabase) String() string { + return fmt.Sprintf("%s (%s)", qdb.QualifiedID(), qdb.Name) +} + +// QualifiedID returns the QualifiedDatabaseID for the database. +func (qdb *QualifiedDatabase) QualifiedID() QualifiedDatabaseID { + return QualifiedDatabaseID{ + OrganizationID: qdb.OrganizationID, + DatabaseID: qdb.ID, + } +} + +// QualifiedDatabaseID is a DatabaseID along with its OrganizationID. +type QualifiedDatabaseID struct { + OrganizationID OrganizationID `json:"org-id"` + DatabaseID DatabaseID `json:"db-id"` +} + +// NewQualifiedDatabaseID is a helper function used to create a +// QualifiedDatabaseID from the provided arguments. +func NewQualifiedDatabaseID(orgID OrganizationID, dbID DatabaseID) QualifiedDatabaseID { + return QualifiedDatabaseID{ + OrganizationID: orgID, + DatabaseID: dbID, + } +} + +// String returns a human-friendly version of the QualifiedDatabaseID. It is only +// used for display purposes; it is not used as any kind of key. For that, see +// the QualifiedDatabaseID.Key() method and the DatabaseKey type. +func (qdbid QualifiedDatabaseID) String() string { + return fmt.Sprintf("[%s:%s]", qdbid.OrganizationID, qdbid.DatabaseID) +} + +// Key returns the string-encoded (delimited by TableKeyDelimiter) globally +// unique DatabaseKey. The key has a prefix because FeatureBase index name +// restrictions require the name to start with a non-numeric value, and since a +// uuid can contain a number as its first character, we have to prefix it with +// something. +func (qdbid QualifiedDatabaseID) Key() DatabaseKey { + if qdbid.DatabaseID == "" { + panic("QualifiedDatabaseID.Key called without an ID set") + } + return DatabaseKey(fmt.Sprintf("%s%s%s%s%s", + PrefixDatabase, + TableKeyDelimiter, + qdbid.OrganizationID, + TableKeyDelimiter, + qdbid.DatabaseID)) +} + +// QualifiedDatabaseIDs is a list of QualifiedDatabaseID. +type QualifiedDatabaseIDs []QualifiedDatabaseID + +func (s QualifiedDatabaseIDs) Len() int { return len(s) } +func (s QualifiedDatabaseIDs) Less(i, j int) bool { + if s[i].OrganizationID != s[j].OrganizationID { + return s[i].OrganizationID < s[j].OrganizationID + } + return s[i].DatabaseID < s[j].DatabaseID +} +func (s QualifiedDatabaseIDs) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + // TableKeyer is an interface implemented by any type which can produce, and be // represented by, a TableKey. In the case of a QualifiedTable, its TableKey // might be something like `tbl__org__db__tableid`, while a general pilosa @@ -129,7 +257,7 @@ func (s StringTableKeyer) Key() TableKey { // TableKey as the value for index.Name. type TableKey string -func (t TableKey) Key() TableKey { return t } +func (tk TableKey) Key() TableKey { return tk } // QualifiedTableID returns the QualifiedTableID based on the key. If TableKey // can't be parsed into a valid (i.e. complete) QualifiedTableID, then blank @@ -138,7 +266,7 @@ func (tk TableKey) QualifiedTableID() QualifiedTableID { qtid, err := QualifiedTableIDFromKey(string(tk)) if err != nil { return NewQualifiedTableID( - NewTableQualifier("", ""), + NewQualifiedDatabaseID("", ""), TableID(tk), ) } @@ -153,7 +281,7 @@ func (s TableKeys) Less(i, j int) bool { return s[i] < s[j] } func (s TableKeys) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // TableID is a table identifier. It is unique within the scope of a -// TableQualifier. Coupled with a TableQualifier, it makes up a +// QualifiedDatabaseID. Coupled with a QualifiedDatabaseID, it makes up a // QualifiedTableID and, when encoded as a string, a TableKey. type TableID string @@ -165,7 +293,7 @@ func (s TableIDs) Less(i, j int) bool { return s[i] < s[j] } func (s TableIDs) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // TableName is a human-friendly string. While it is not used as a primary key, -// uniqueness is generally enforced within the scope of a TableQualifier. +// uniqueness is generally enforced within the scope of a QualifiedDatabaseID. type TableName string // TableNames is a sortable slice of TableName. @@ -224,7 +352,7 @@ func (t *Table) CreateID() (TableID, error) { } // NewTable returns a new instance of table with a pseudo-random ID which is -// assumed to be unique within the scope of a TableQualifier. +// assumed to be unique within the scope of a QualifiedDatabaseID. func NewTable(name TableName) *Table { return &Table{ Name: name, @@ -315,84 +443,6 @@ func (o Tables) Len() int { return len(o) } func (o Tables) Less(i, j int) bool { return o[i].Name < o[j].Name } func (o Tables) Swap(i, j int) { o[i], o[j] = o[j], o[i] } -// TableQualifierKey is the unique TableQualifier values encoded as a string. The -// current encoding is delimited as `prefix|OrganizationID|DatabaseID` (where -// the pipe may be some other delimiter) by the TableQualifier.Key() method. -type TableQualifierKey string - -// Qualifier returns the Qualifier based on the values encoded into the -// TableQualifierKey string. -func (tqk TableQualifierKey) Qualifier() TableQualifier { - parts := strings.Split(string(tqk), TableKeyDelimiter) - - if len(parts) < 3 { - return NewTableQualifier("", "") - } - - return NewTableQualifier( - OrganizationID(parts[1]), - DatabaseID(parts[2]), - ) -} - -// OrganizationID returns the OrganizationID value that has been encoded into -// the TableQualifierKey string. -func (tqk TableQualifierKey) OrganizationID() OrganizationID { - parts := strings.Split(string(tqk), TableKeyDelimiter) - - if len(parts) < 2 { - return "" - } - - return OrganizationID(parts[1]) -} - -// DatabaseID returns the DatabaseID value that has been encoded into the -// TableQualifierKey string. -func (tqk TableQualifierKey) DatabaseID() DatabaseID { - parts := strings.Split(string(tqk), TableKeyDelimiter) - - if len(parts) < 3 { - return "" - } - - return DatabaseID(parts[2]) -} - -// TableQualifier contains all the elements required to fully qualify a table. -type TableQualifier struct { - OrganizationID OrganizationID `json:"org-id"` - DatabaseID DatabaseID `json:"db-id"` -} - -// NewTableQualifier is a helper function used to create a TableQualifier from -// the provided arguments. -func NewTableQualifier(orgID OrganizationID, dbID DatabaseID) TableQualifier { - return TableQualifier{ - OrganizationID: orgID, - DatabaseID: dbID, - } -} - -// String returns a human-friendly version of the TableQualifier. It is only -// used for display purposes; it is not used as any kind of key. For that, see -// the TableQualifier.Key() method and the TableQualifierKey type. -func (tq TableQualifier) String() string { - return fmt.Sprintf("[%s:%s]", tq.OrganizationID, tq.DatabaseID) -} - -// Key returns the string-encoded (delimited by TableKeyDelimiter) -// TableQualifierKey. -func (tq TableQualifier) Key() TableQualifierKey { - return TableQualifierKey(fmt.Sprintf("%s%s%s%s%s", - PrefixTable, - TableKeyDelimiter, - tq.OrganizationID, - TableKeyDelimiter, - tq.DatabaseID, - )) -} - //////////////////////////////////////////////// // QualifiedTableID is a globally unique table identifier. It is a @@ -400,17 +450,17 @@ func (tq TableQualifier) Key() TableQualifierKey { // portion). Most things will take a Name or an ID and do the right // thing™. type QualifiedTableID struct { - TableQualifier + QualifiedDatabaseID ID TableID `json:"id"` Name TableName `json:"name"` } // NewQualifiedTableID is a helper function used to create a QualifiedTableID // from the provided arguments. -func NewQualifiedTableID(q TableQualifier, id TableID) QualifiedTableID { +func NewQualifiedTableID(qdbid QualifiedDatabaseID, tid TableID) QualifiedTableID { return QualifiedTableID{ - TableQualifier: q, - ID: id, + QualifiedDatabaseID: qdbid, + ID: tid, } } @@ -422,7 +472,7 @@ func QualifiedTableIDFromKey(key string) (QualifiedTableID, error) { case 4: // prefix|orgID|dbID|tblID return NewQualifiedTableID( - NewTableQualifier( + NewQualifiedDatabaseID( OrganizationID(parts[1]), DatabaseID(parts[2]), ), @@ -433,14 +483,31 @@ func QualifiedTableIDFromKey(key string) (QualifiedTableID, error) { } } -// String returns a human-friendly version of the TableQualifierID. It is only -// used for display purposes; it is not used as any kind of key. For that, see -// the TableQualifierID.Key() method. +// QualifiedDatabaseIDFromKey decodes a string key into a QualifiedDatabaseID. +// The key is assumed to have been encoded using the QualifiedDatabaseID.Key() +// method. +func QualifiedDatabaseIDFromKey(key string) (QualifiedDatabaseID, error) { + parts := strings.Split(key, TableKeyDelimiter) + switch len(parts) { + case 3: + // prefix|orgID|dbID + return NewQualifiedDatabaseID( + OrganizationID(parts[1]), + DatabaseID(parts[2]), + ), nil + default: + return QualifiedDatabaseID{}, errors.Errorf("invalid key: %s", key) + } +} + +// String returns a human-friendly version of the QualifiedDatabaseID. It is +// only used for display purposes; it is not used as any kind of key. For that, +// see the QualifiedDatabaseID.Key() method. func (qtid QualifiedTableID) String() string { if qtid.ID == "" { - return fmt.Sprintf("%s%s", qtid.TableQualifier, qtid.Name) + return fmt.Sprintf("%s%s", qtid.QualifiedDatabaseID, qtid.Name) } - return fmt.Sprintf("%s%s", qtid.TableQualifier, qtid.ID) + return fmt.Sprintf("%s%s", qtid.QualifiedDatabaseID, qtid.ID) } // Key returns the string-encoded (delimited by TableKeyDelimiter) globally @@ -452,35 +519,49 @@ func (qtid QualifiedTableID) Key() TableKey { if qtid.ID == "" { panic("QualifiedTableID.Key called without an ID set") } - return TableKey(fmt.Sprintf("%s%s%s", - qtid.TableQualifier.Key(), + return TableKey(fmt.Sprintf("%s%s%s%s%s%s%s", + PrefixTable, + TableKeyDelimiter, + qtid.OrganizationID, + TableKeyDelimiter, + qtid.DatabaseID, TableKeyDelimiter, qtid.ID)) } // Equals returns true if `other` is the same as qtid. Note: the `Name` value is -// ignored in this comparison; only `TableQualifier` and `ID` are considered. +// ignored in this comparison; only `QualifiedDatabaseID` and `ID` are +// considered. func (qtid QualifiedTableID) Equals(other QualifiedTableID) bool { - if qtid.TableQualifier == other.TableQualifier && qtid.ID == other.ID { + if qtid.QualifiedDatabaseID == other.QualifiedDatabaseID && qtid.ID == other.ID { return true } return false } +// Qualifier returns the QualifiedDatabaseID (qdbid) portion of the +// QualifiedTableID (qtid). +func (qtid QualifiedTableID) Qualifier() QualifiedDatabaseID { + return QualifiedDatabaseID{ + OrganizationID: qtid.OrganizationID, + DatabaseID: qtid.DatabaseID, + } +} + //////////////////////////////////////////////// -// QualifiedTable wraps Table and includes a TableQualifier. +// QualifiedTable wraps Table and includes a QualifiedDatabaseID. type QualifiedTable struct { + QualifiedDatabaseID Table - TableQualifier } // NewQualifiedTable returns the tbl as a QualifiedTable with the provided -// TableQualifier. -func NewQualifiedTable(qual TableQualifier, tbl *Table) *QualifiedTable { +// QualifiedDatabaseID. +func NewQualifiedTable(qdbid QualifiedDatabaseID, tbl *Table) *QualifiedTable { return &QualifiedTable{ - Table: *tbl, - TableQualifier: qual, + QualifiedDatabaseID: qdbid, + Table: *tbl, } } @@ -496,17 +577,17 @@ func (qt QualifiedTable) String() string { return fmt.Sprintf("%s (%s)", qt.QualifiedID(), qt.Name) } -// Qualifier returns the TableQualifier portion of the QualifiedTable. -func (qt *QualifiedTable) Qualifier() TableQualifier { - return qt.TableQualifier +// Qualifier returns the QualifiedDatabaseID portion of the QualifiedTable. +func (qt *QualifiedTable) Qualifier() QualifiedDatabaseID { + return qt.QualifiedDatabaseID } // QualifiedID returns the QualifiedTableID for the table. func (qt *QualifiedTable) QualifiedID() QualifiedTableID { return QualifiedTableID{ - TableQualifier: qt.TableQualifier, - ID: qt.ID, - Name: qt.Name, + QualifiedDatabaseID: qt.QualifiedDatabaseID, + ID: qt.ID, + Name: qt.Name, } } diff --git a/dax/table_test.go b/dax/table_test.go index 091722cdd..5706ccf9d 100644 --- a/dax/table_test.go +++ b/dax/table_test.go @@ -313,17 +313,17 @@ func TestTable(t *testing.T) { t.Run("New", func(t *testing.T) { tbl := dax.NewTable(tableName) tbl.CreateID() - qual := dax.TableQualifier{ + qdbid := dax.QualifiedDatabaseID{ OrganizationID: orgID, DatabaseID: dbID, } - qtbl := dax.NewQualifiedTable(qual, tbl) + qtbl := dax.NewQualifiedTable(qdbid, tbl) assert.NotEmpty(t, qtbl.ID) assert.Equal(t, tbl.ID, qtbl.ID) tq := qtbl.Qualifier() - assert.Equal(t, qual.OrganizationID, tq.OrganizationID) - assert.Equal(t, qual.DatabaseID, tq.DatabaseID) + assert.Equal(t, qdbid.OrganizationID, tq.OrganizationID) + assert.Equal(t, qdbid.DatabaseID, tq.DatabaseID) wrappedTable := qtbl.Table assert.Equal(t, tbl.Name, wrappedTable.Name) @@ -345,11 +345,11 @@ func TestTable(t *testing.T) { t.Run("ToJSON", func(t *testing.T) { tbl := dax.NewTable(tableName) tbl.CreateID() - qual := dax.TableQualifier{ + qdbid := dax.QualifiedDatabaseID{ OrganizationID: orgID, DatabaseID: dbID, } - qtbl := dax.NewQualifiedTable(qual, tbl) + qtbl := dax.NewQualifiedTable(qdbid, tbl) id := qtbl.ID b, err := json.Marshal(qtbl) diff --git a/dax/test/dax/dax_test.go b/dax/test/dax/dax_test.go index 87b510a3b..823f8c1ac 100644 --- a/dax/test/dax/dax_test.go +++ b/dax/test/dax/dax_test.go @@ -26,7 +26,18 @@ func TestDAXIntegration(t *testing.T) { t.Skip("skipping integration test") } - qual := dax.NewTableQualifier("acme", "db1") + qdbid := dax.NewQualifiedDatabaseID("acme", "db1") + qdb := &dax.QualifiedDatabase{ + OrganizationID: qdbid.OrganizationID, + Database: dax.Database{ + ID: qdbid.DatabaseID, + Name: "dbname1", + Options: dax.DatabaseOptions{ + WorkersMin: 1, + WorkersMax: 1, + }, + }, + } t.Run("ServiceStart", func(t *testing.T) { t.Run("AllServicesByDefault", func(t *testing.T) { @@ -104,6 +115,14 @@ func TestDAXIntegration(t *testing.T) { svcmgr := mc.Manage() + // Set up MDS client. + mdsClient := mdsclient.New(svcmgr.MDS.Address(), svcmgr.Logger) + + // Create database. + qdb.Options.WorkersMin = 1 + qdb.Options.WorkersMax = 1 + assert.NoError(t, mdsClient.CreateDatabase(context.Background(), qdb)) + // skips is a list of tests which are currently not passing in dax. We // need to get these passing before alpha. skips := []string{ @@ -160,7 +179,7 @@ func TestDAXIntegration(t *testing.T) { runTableTests(t, svcmgr.Queryer.Address(), - basicTableTestConfig(qual, tableTests...)..., + basicTableTestConfig(qdbid, tableTests...)..., ) }) @@ -168,9 +187,19 @@ func TestDAXIntegration(t *testing.T) { mc := test.MustRunManagedCommand(t) defer mc.Close() + svcmgr := mc.Manage() + + // Set up MDS client. + mdsClient := mdsclient.New(svcmgr.MDS.Address(), svcmgr.Logger) + + // Create database. + qdb.Options.WorkersMin = 1 + qdb.Options.WorkersMax = 1 + assert.NoError(t, mdsClient.CreateDatabase(context.Background(), qdb)) + runTableTests(t, mc.Manage().Queryer.Address(), - basicTableTestConfig(qual, defs.Keyed)..., + basicTableTestConfig(qdbid, defs.Keyed)..., ) }) @@ -182,6 +211,14 @@ func TestDAXIntegration(t *testing.T) { svcmgr := mc.Manage() + // Set up MDS client. + mdsClient := mdsclient.New(svcmgr.MDS.Address(), svcmgr.Logger) + + // Create database. + qdb.Options.WorkersMin = 2 + qdb.Options.WorkersMax = 2 + assert.NoError(t, mdsClient.CreateDatabase(context.Background(), qdb)) + computers := svcmgr.Computers() computerKey0 := dax.ServiceKey(dax.ServicePrefixComputer + "0") computerKey1 := dax.ServiceKey(dax.ServicePrefixComputer + "1") @@ -189,13 +226,10 @@ func TestDAXIntegration(t *testing.T) { // Ingest and query some data. runTableTests(t, svcmgr.Queryer.Address(), - basicTableTestConfig(qual, defs.Keyed)..., + basicTableTestConfig(qdbid, defs.Keyed)..., ) - // Set up MDS client. - mdsClient := mdsclient.New(svcmgr.MDS.Address(), svcmgr.Logger) - - qtid, err := mdsClient.TableID(context.Background(), qual, dax.TableName(defs.Keyed.Name(0))) + qtid, err := mdsClient.TableID(context.Background(), qdbid, dax.TableName(defs.Keyed.Name(0))) assert.NoError(t, err) // ensure partitions are covered @@ -238,13 +272,21 @@ func TestDAXIntegration(t *testing.T) { svcmgr := mc.Manage() + // Set up MDS client. + mdsClient := mdsclient.New(svcmgr.MDS.Address(), svcmgr.Logger) + + // Create database. + qdb.Options.WorkersMin = 1 + qdb.Options.WorkersMax = 1 + assert.NoError(t, mdsClient.CreateDatabase(context.Background(), qdb)) + computerKey0 := dax.ServiceKey(dax.ServicePrefixComputer + "0") // Ingest and query some data. t.Run("ingest and query some data", func(t *testing.T) { runTableTests(t, svcmgr.Queryer.Address(), - basicTableTestConfig(qual, defs.Keyed)..., + basicTableTestConfig(qdbid, defs.Keyed)..., ) }) @@ -268,7 +310,7 @@ func TestDAXIntegration(t *testing.T) { runTableTests(t, svcmgr.Queryer.Address(), tableTestConfig{ - qual: qual, + qdbid: qdbid, test: defs.Keyed, skipCreate: true, skipInsert: true, @@ -284,6 +326,14 @@ func TestDAXIntegration(t *testing.T) { svcmgr := mc.Manage() + // Set up MDS client. + mdsClient := mdsclient.New(svcmgr.MDS.Address(), svcmgr.Logger) + + // Create database. + qdb.Options.WorkersMin = 1 + qdb.Options.WorkersMax = 1 + assert.NoError(t, mdsClient.CreateDatabase(context.Background(), qdb)) + computerKey0 := dax.ServiceKey(dax.ServicePrefixComputer + "0") // Ingest and query some data. @@ -291,19 +341,16 @@ func TestDAXIntegration(t *testing.T) { runTableTests(t, svcmgr.Queryer.Address(), tableTestConfig{ - qual: qual, + qdbid: qdbid, test: defs.Keyed, insertSet: 0, }, ) }) - // Set up MDS client. - mdsClient := mdsclient.New(svcmgr.MDS.Address(), svcmgr.Logger) - // Snapshot table ctx := context.Background() - qtid, err := mdsClient.TableID(ctx, qual, dax.TableName(defs.Keyed.Name(0))) + qtid, err := mdsClient.TableID(ctx, qdbid, dax.TableName(defs.Keyed.Name(0))) assert.NoError(t, err) mdsClient.SnapshotTable(ctx, qtid) @@ -313,7 +360,7 @@ func TestDAXIntegration(t *testing.T) { runTableTests(t, svcmgr.Queryer.Address(), tableTestConfig{ - qual: qual, + qdbid: qdbid, test: defs.Keyed, skipCreate: true, insertSet: 1, @@ -341,7 +388,7 @@ func TestDAXIntegration(t *testing.T) { runTableTests(t, svcmgr.Queryer.Address(), tableTestConfig{ - qual: qual, + qdbid: qdbid, test: defs.Keyed, skipCreate: true, skipInsert: true, @@ -357,6 +404,14 @@ func TestDAXIntegration(t *testing.T) { svcmgr := mc.Manage() + // Set up MDS client. + mdsClient := mdsclient.New(svcmgr.MDS.Address(), svcmgr.Logger) + + // Create database. + qdb.Options.WorkersMin = 1 + qdb.Options.WorkersMax = 1 + assert.NoError(t, mdsClient.CreateDatabase(context.Background(), qdb)) + computerKey0 := dax.ServiceKey(dax.ServicePrefixComputer + "0") mdsKey := dax.ServiceKey(dax.ServicePrefixMDS) @@ -365,13 +420,10 @@ func TestDAXIntegration(t *testing.T) { // Ingest and query some data. runTableTests(t, svcmgr.Queryer.Address(), - basicTableTestConfig(qual, defs.Keyed)..., + basicTableTestConfig(qdbid, defs.Keyed)..., ) - // Set up MDS client. - mdsClient := mdsclient.New(svcmgr.MDS.Address(), svcmgr.Logger) - - qtid, err := mdsClient.TableID(context.Background(), qual, dax.TableName(defs.Keyed.Name(0))) + qtid, err := mdsClient.TableID(context.Background(), qdbid, dax.TableName(defs.Keyed.Name(0))) assert.NoError(t, err) // ensure partitions are covered @@ -410,13 +462,21 @@ func TestDAXIntegration(t *testing.T) { svcmgr := mc.Manage() + // Set up MDS client. + mdsClient := mdsclient.New(svcmgr.MDS.Address(), svcmgr.Logger) + + // Create database. + qdb.Options.WorkersMin = 1 + qdb.Options.WorkersMax = 1 + assert.NoError(t, mdsClient.CreateDatabase(context.Background(), qdb)) + computerKey0 := dax.ServiceKey(dax.ServicePrefixComputer + "0") // Ingest and query some data. t.Run("ingest and query some data", func(t *testing.T) { runTableTests(t, svcmgr.Queryer.Address(), - basicTableTestConfig(qual, defs.Keyed)..., + basicTableTestConfig(qdbid, defs.Keyed)..., ) }) @@ -439,7 +499,7 @@ func TestDAXIntegration(t *testing.T) { runTableTests(t, svcmgr.Queryer.Address(), tableTestConfig{ - qual: qual, + qdbid: qdbid, test: defs.Keyed, skipCreate: true, skipInsert: true, @@ -452,7 +512,7 @@ func TestDAXIntegration(t *testing.T) { /////////////////////////////////////////////////// type tableTestConfig struct { - qual dax.TableQualifier + qdbid dax.QualifiedDatabaseID test defs.TableTest skipCreate bool skipInsert bool @@ -461,20 +521,19 @@ type tableTestConfig struct { querySet int } -func basicTableTestConfig(qual dax.TableQualifier, tests ...defs.TableTest) []tableTestConfig { +func basicTableTestConfig(qdbid dax.QualifiedDatabaseID, tests ...defs.TableTest) []tableTestConfig { ret := make([]tableTestConfig, len(tests)) for i := range tests { ret[i] = tableTestConfig{ - qual: qual, - test: tests[i], + qdbid: qdbid, + test: tests[i], } } return ret } -// func runTableTests(t *testing.T, queryerAddr dax.Address, qual dax.TableQualifier, doCreate bool, tests ...defs.TableTest) { func runTableTests(t *testing.T, queryerAddr dax.Address, cfgs ...tableTestConfig) { emptyWireQueryResponse := &featurebase.WireQueryResponse{ Schema: featurebase.WireQuerySchema{ @@ -491,7 +550,7 @@ func runTableTests(t *testing.T, queryerAddr dax.Address, cfgs ...tableTestConfi if !cfg.skipCreate { // Create a table. if cfg.test.HasTable() { - resp := runSQL(t, queryerAddr, cfg.qual, cfg.test.CreateTable()) + resp := runSQL(t, queryerAddr, cfg.qdbid, cfg.test.CreateTable()) assertResponseEqual(t, emptyWireQueryResponse, resp) } } @@ -499,7 +558,7 @@ func runTableTests(t *testing.T, queryerAddr dax.Address, cfgs ...tableTestConfi if !cfg.skipInsert { // Populate table with data. if cfg.test.HasTable() && cfg.test.HasData() { - resp := runSQL(t, queryerAddr, cfg.qual, cfg.test.InsertInto(t, cfg.insertSet)) + resp := runSQL(t, queryerAddr, cfg.qdbid, cfg.test.InsertInto(t, cfg.insertSet)) assertResponseEqual(t, emptyWireQueryResponse, resp) } } @@ -525,7 +584,7 @@ func runTableTests(t *testing.T, queryerAddr dax.Address, cfgs ...tableTestConfi expRows = sqltest.ExpRowsPlus1[cfg.querySet-1] } - resp := runSQL(t, queryerAddr, cfg.qual, sql) + resp := runSQL(t, queryerAddr, cfg.qdbid, sql) headers := resp.Schema.Fields rows := resp.Data var err error @@ -602,7 +661,7 @@ func runTableTests(t *testing.T, queryerAddr dax.Address, cfgs ...tableTestConfi expRows = pqltest.ExpRowsPlus1[cfg.querySet-1] } - resp := runPQL(t, queryerAddr, cfg.qual, pqltest.Table, pql) + resp := runPQL(t, queryerAddr, cfg.qdbid, pqltest.Table, pql) headers := resp.Schema.Fields rows := resp.Data var err error @@ -680,23 +739,23 @@ func (c *wireResponseComparer) Equal() bool { return assert.Equal(c.tb, c.exp, c.got) } -func runSQL(tb testing.TB, queryerAddr dax.Address, qual dax.TableQualifier, sql string) *featurebase.WireQueryResponse { +func runSQL(tb testing.TB, queryerAddr dax.Address, qdbid dax.QualifiedDatabaseID, sql string) *featurebase.WireQueryResponse { tb.Helper() client := queryerclient.New(queryerAddr, logger.StderrLogger) - resp, err := client.QuerySQL(context.Background(), qual, sql) + resp, err := client.QuerySQL(context.Background(), qdbid, sql) assert.NoError(tb, err) return resp } -func runPQL(tb testing.TB, queryerAddr dax.Address, qual dax.TableQualifier, table string, pql string) *featurebase.WireQueryResponse { +func runPQL(tb testing.TB, queryerAddr dax.Address, qdbid dax.QualifiedDatabaseID, table string, pql string) *featurebase.WireQueryResponse { tb.Helper() client := queryerclient.New(queryerAddr, logger.StderrLogger) - resp, err := client.QueryPQL(context.Background(), qual, dax.TableName(table), pql) + resp, err := client.QueryPQL(context.Background(), qdbid, dax.TableName(table), pql) assert.NoError(tb, err) return resp diff --git a/dax/test/table.go b/dax/test/table.go index 8d2ba93e8..2bde25a40 100644 --- a/dax/test/table.go +++ b/dax/test/table.go @@ -11,7 +11,7 @@ import ( // general configuration. This function creates a Table with a random TableID. // If you need to specify the TableID yourself, use the TestQualifiedTableWithID // function. -func TestQualifiedTable(t *testing.T, qual dax.TableQualifier, name dax.TableName, partitionN int, keyed bool) *dax.QualifiedTable { +func TestQualifiedTable(t *testing.T, qdbid dax.QualifiedDatabaseID, name dax.TableName, partitionN int, keyed bool) *dax.QualifiedTable { t.Helper() var pkFieldType dax.BaseType @@ -22,7 +22,6 @@ func TestQualifiedTable(t *testing.T, qual dax.TableQualifier, name dax.TableNam } tbl := dax.NewTable(name) - tbl.CreateID() tbl.PartitionN = partitionN tbl.Fields = []*dax.Field{ { @@ -32,14 +31,33 @@ func TestQualifiedTable(t *testing.T, qual dax.TableQualifier, name dax.TableNam } return dax.NewQualifiedTable( - qual, + qdbid, tbl, ) } +// TestQualifiedDatabaseWithID is a test helper function for creating a database +// based on a general configuration, and having the specified DatabaseID. +func TestQualifiedDatabaseWithID(t *testing.T, orgID dax.OrganizationID, id dax.DatabaseID, name dax.DatabaseName, opts dax.DatabaseOptions) *dax.QualifiedDatabase { + t.Helper() + + db := dax.Database{ + ID: dax.DatabaseID(id), + Name: name, + Options: opts, + } + + qdb := &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: db, + } + + return qdb +} + // TestQualifiedTableWithID is a test helper function for creating a table based // on a general configuration, and having the specified TableID. -func TestQualifiedTableWithID(t *testing.T, qual dax.TableQualifier, id string, name dax.TableName, partitionN int, keyed bool) *dax.QualifiedTable { +func TestQualifiedTableWithID(t *testing.T, qdbid dax.QualifiedDatabaseID, id string, name dax.TableName, partitionN int, keyed bool) *dax.QualifiedTable { t.Helper() var pkFieldType dax.BaseType @@ -63,7 +81,7 @@ func TestQualifiedTableWithID(t *testing.T, qual dax.TableQualifier, id string, } return dax.NewQualifiedTable( - qual, + qdbid, tbl, ) } diff --git a/dax/transaction.go b/dax/transaction.go new file mode 100644 index 000000000..0e8e0f598 --- /dev/null +++ b/dax/transaction.go @@ -0,0 +1,9 @@ +package dax + +import "context" + +type Transaction interface { + Commit() error + Context() context.Context + Rollback() error +} diff --git a/dax/workerjob.go b/dax/workerjob.go index a249bd82b..2e0c7b12b 100644 --- a/dax/workerjob.go +++ b/dax/workerjob.go @@ -5,44 +5,39 @@ import ( "strings" ) -// Worker is a generic identifier used to represent a service responsible for -// doing certain jobs. In the case of dax, this is typically the Address of a -// compute or translate node. Services such as the Balancer use Workers (as -// opposed to specifically using Address) in order to remain generic, and to -// keep the business logic between services slightly less coupled. -type Worker string - -// Workers is a sortable slice of Worker. -type Workers []Worker - -func (w Workers) Len() int { return len(w) } -func (w Workers) Less(i, j int) bool { return w[i] < w[j] } -func (w Workers) Swap(i, j int) { w[i], w[j] = w[j], w[i] } - // Job is a generic identifier used to represent a specific role assigned to a // worker. type Job string +// Job allows a Job to implement the Jobber interface. +func (j Job) Job() Job { + return j +} + // Jobs is a slice of Job. type Jobs []Job +type Jobber interface { + Job() Job +} + // WorkerInfo represents a Worker and the Jobs to which it has been assigned. type WorkerInfo struct { - ID Worker - Jobs []Job + Address Address + Jobs []Job } // WorkerInfos is a sortable slice of WorkerInfo. type WorkerInfos []WorkerInfo func (w WorkerInfos) Len() int { return len(w) } -func (w WorkerInfos) Less(i, j int) bool { return w[i].ID < w[j].ID } +func (w WorkerInfos) Less(i, j int) bool { return w[i].Address < w[j].Address } func (w WorkerInfos) Swap(i, j int) { w[i], w[j] = w[j], w[i] } // WorkerDiff represents the changes made to a Worker following the latest // event. type WorkerDiff struct { - WorkerID Worker + Address Address AddedJobs []Job RemovedJobs []Job } @@ -51,7 +46,7 @@ type WorkerDiff struct { // ID. Any job that is added and then removed or removed and then // added cancels out and won't be present after add is called. func (w *WorkerDiff) Add(w2 WorkerDiff) { - if w.WorkerID != w2.WorkerID { + if w.Address != w2.Address { panic("can't add worker diffs from different workers") } a1 := NewSet(w.AddedJobs...) @@ -74,7 +69,7 @@ func (w *WorkerDiff) Add(w2 WorkerDiff) { type WorkerDiffs []WorkerDiff func (w WorkerDiffs) Len() int { return len(w) } -func (w WorkerDiffs) Less(i, j int) bool { return w[i].WorkerID < w[j].WorkerID } +func (w WorkerDiffs) Less(i, j int) bool { return w[i].Address < w[j].Address } func (w WorkerDiffs) Swap(i, j int) { w[i], w[j] = w[j], w[i] } // Set is a set of stringy items. @@ -109,7 +104,8 @@ func (s Set[K]) Remove(k K) { delete(s, k) } -func (s Set[K]) RemovePrefix(prefix string) []K { +// RemoveByPrefix removes all items from Set that have the given prefix. +func (s Set[K]) RemoveByPrefix(prefix string) []K { ret := make([]K, 0) for k := range s { if strings.HasPrefix(string(k), prefix) { diff --git a/idk/ingest.go b/idk/ingest.go index 47d33280d..f3cf02485 100644 --- a/idk/ingest.go +++ b/idk/ingest.go @@ -1007,10 +1007,10 @@ func (m *Main) setupClient() (*tls.Config, error) { // MDS doesn't auto-create a table based on IDK ingest; the table must // already exist. mdsClient := mdsclient.New(dax.Address(m.MDSAddress), m.log) - qual := dax.NewTableQualifier(m.OrganizationID, m.DatabaseID) - qtid, err := mdsClient.TableID(ctx, qual, m.TableName) + qdbid := dax.NewQualifiedDatabaseID(m.OrganizationID, m.DatabaseID) + qtid, err := mdsClient.TableID(ctx, qdbid, m.TableName) if err != nil { - return nil, errors.Wrapf(err, "getting table id: qual: %s, table name: %s", qual, m.TableName) + return nil, errors.Wrapf(err, "getting table id: qual: %s, table name: %s", qdbid, m.TableName) } qtbl, err := mdsClient.Table(ctx, qtid) if err != nil { @@ -1019,7 +1019,7 @@ func (m *Main) setupClient() (*tls.Config, error) { m.Qtbl = qtbl m.Index = string(qtbl.Key()) - m.SchemaManager = mds.NewSchemaManager(dax.Address(m.MDSAddress), qual, m.log) + m.SchemaManager = mds.NewSchemaManager(dax.Address(m.MDSAddress), qdbid, m.log) m.NewImporterFn = func() pilosacore.Importer { return mds.NewImporter(mdsClient, mdsClient, qtbl.Qualifier(), &qtbl.Table) diff --git a/idk/ingest_test.go b/idk/ingest_test.go index 7a25000f2..64aab66e2 100644 --- a/idk/ingest_test.go +++ b/idk/ingest_test.go @@ -58,7 +58,7 @@ func configureTestFlagsMDS(main *Main, address dax.Address, qtbl *dax.QualifiedT mdsClient := mdsclient.New(dax.Address(address), logger.StderrLogger) main.NewImporterFn = func() pilosa.Importer { - return mds.NewImporter(mdsClient, mdsClient, qtbl.TableQualifier, &qtbl.Table) + return mds.NewImporter(mdsClient, mdsClient, qtbl.QualifiedDatabaseID, &qtbl.Table) } } @@ -1756,6 +1756,24 @@ func TestBatchTargetMDS(t *testing.T) { orgID := dax.OrganizationID("acme") dbID := dax.DatabaseID("db1") + mdsClient := mdsclient.New(mdsAddress, logger.StderrLogger) + + ctx := context.Background() + + // Create the database. + qdb := &dax.QualifiedDatabase{ + OrganizationID: orgID, + Database: dax.Database{ + ID: dbID, + Name: "dbname1", + Options: dax.DatabaseOptions{ + WorkersMin: 1, + WorkersMax: 1, + }, + }, + } + mdsClient.CreateDatabase(ctx, qdb) + t.Run("FieldTypes", func(t *testing.T) { tests := []struct { fieldType dax.BaseType @@ -1859,14 +1877,11 @@ func TestBatchTargetMDS(t *testing.T) { } qtbl := dax.NewQualifiedTable( - dax.NewTableQualifier(orgID, dbID), + dax.NewQualifiedDatabaseID(orgID, dbID), tbl, ) - ctx := context.Background() - // Create the table in MDS Schemar. - mdsClient := mdsclient.New(mdsAddress, logger.StderrLogger) if err := mdsClient.CreateTable(ctx, qtbl); err != nil { t.Fatalf("creating table: %v", err) } diff --git a/idk/mds/importer.go b/idk/mds/importer.go index f70c96c34..0d7a1a31c 100644 --- a/idk/mds/importer.go +++ b/idk/mds/importer.go @@ -21,16 +21,16 @@ type importer struct { noder dax.Noder schemar dax.Schemar - mu sync.Mutex - qual dax.TableQualifier - tbl *dax.Table + mu sync.Mutex + qdbid dax.QualifiedDatabaseID + tbl *dax.Table } -func NewImporter(noder dax.Noder, schemar dax.Schemar, qual dax.TableQualifier, tbl *dax.Table) *importer { +func NewImporter(noder dax.Noder, schemar dax.Schemar, qdbid dax.QualifiedDatabaseID, tbl *dax.Table) *importer { return &importer{ noder: noder, schemar: schemar, - qual: qual, + qdbid: qdbid, tbl: tbl, } } @@ -261,10 +261,10 @@ func (m *importer) getQtbl(ctx context.Context, tid dax.TableID) (*dax.Qualified defer m.mu.Unlock() if m.tbl != nil { - return dax.NewQualifiedTable(m.qual, m.tbl), nil + return dax.NewQualifiedTable(m.qdbid, m.tbl), nil } - qtid := dax.NewQualifiedTableID(m.qual, tid) + qtid := dax.NewQualifiedTableID(m.qdbid, tid) qtbl, err := m.schemar.TableByID(ctx, qtid) if err != nil { diff --git a/idk/mds/schemamanager.go b/idk/mds/schemamanager.go index b73fe4537..d1e691c06 100644 --- a/idk/mds/schemamanager.go +++ b/idk/mds/schemamanager.go @@ -18,14 +18,14 @@ import ( // schemaManager type schemaManager struct { client *mdsclient.Client - qual dax.TableQualifier + qdbid dax.QualifiedDatabaseID logger logger.Logger } -func NewSchemaManager(mdsAddress dax.Address, qual dax.TableQualifier, logger logger.Logger) *schemaManager { +func NewSchemaManager(mdsAddress dax.Address, qdbid dax.QualifiedDatabaseID, logger logger.Logger) *schemaManager { return &schemaManager{ client: mdsclient.New(mdsAddress, logger), - qual: qual, + qdbid: qdbid, logger: logger, } } @@ -41,7 +41,7 @@ func (s *schemaManager) Schema() (*featurebase_client.Schema, error) { // method returns. schema := featurebase_client.NewSchema() - tables, err := s.client.Tables(context.Background(), s.qual) + tables, err := s.client.Tables(context.Background(), s.qdbid) if err != nil { return nil, err } diff --git a/sql3/planner/compilecreateview.go b/sql3/planner/compilecreateview.go index d3a554ee7..b056a8e52 100644 --- a/sql3/planner/compilecreateview.go +++ b/sql3/planner/compilecreateview.go @@ -3,8 +3,8 @@ package planner import ( - "github.com/molecula/featurebase/v3/sql3/parser" - "github.com/molecula/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" ) // compileCreateViewStatement compiles a parser.CreateViewStatement AST into a PlanOperator diff --git a/sql3/planner/compiledropview.go b/sql3/planner/compiledropview.go index 819c84fac..fec75a82b 100644 --- a/sql3/planner/compiledropview.go +++ b/sql3/planner/compiledropview.go @@ -3,9 +3,9 @@ package planner import ( - "github.com/molecula/featurebase/v3/sql3" - "github.com/molecula/featurebase/v3/sql3/parser" - "github.com/molecula/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" ) // compileDropViewStatement compiles a DROP VIEW statement into a PlanOperator. diff --git a/sql3/planner/opalterview.go b/sql3/planner/opalterview.go index 70789acb1..5bf3f9832 100644 --- a/sql3/planner/opalterview.go +++ b/sql3/planner/opalterview.go @@ -6,8 +6,8 @@ import ( "context" "fmt" - "github.com/molecula/featurebase/v3/sql3" - "github.com/molecula/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" ) // PlanOpAlterView implements the ALTER VIEW operator diff --git a/sql3/planner/opcreateview.go b/sql3/planner/opcreateview.go index 310a6bdfa..5d24656e9 100644 --- a/sql3/planner/opcreateview.go +++ b/sql3/planner/opcreateview.go @@ -6,9 +6,9 @@ import ( "context" "fmt" - "github.com/molecula/featurebase/v3/dax" - "github.com/molecula/featurebase/v3/sql3" - "github.com/molecula/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" ) // PlanOpCreateView implements the CREATE VIEW operator diff --git a/sql3/planner/opdropview.go b/sql3/planner/opdropview.go index c90b50469..341034f8d 100644 --- a/sql3/planner/opdropview.go +++ b/sql3/planner/opdropview.go @@ -6,8 +6,8 @@ import ( "context" "fmt" - "github.com/molecula/featurebase/v3/sql3" - "github.com/molecula/featurebase/v3/sql3/planner/types" + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" ) // PlanOpDropView plan operator to drop a view. diff --git a/sql3/planner/opfeaturebasetables.go b/sql3/planner/opfeaturebasetables.go index 59b03375d..2eae4c2cc 100644 --- a/sql3/planner/opfeaturebasetables.go +++ b/sql3/planner/opfeaturebasetables.go @@ -135,11 +135,17 @@ func (i *showTablesRowIter) Next(ctx context.Context) (types.Row, error) { } default: u := i.planner.systemAPI.DataDir() - u = fmt.Sprintf("%s/indexes/%s", u, indexName) - spaceUsed, err = pilosa.GetDiskUsage(u) - if err != nil { - return nil, err + // TODO(tlt): GetDiskUsage needs to be behind an interface because + // this doesn't work in serverless. For now I'm just going to skip + // it based on the emtpy DataDir, but let's do this the right way. + if u != "" { + u = fmt.Sprintf("%s/indexes/%s", u, indexName) + + spaceUsed, err = pilosa.GetDiskUsage(u) + if err != nil { + return nil, err + } } } diff --git a/sql3/planner/systemobjects.go b/sql3/planner/systemobjects.go index d806d7359..909a6e1e0 100644 --- a/sql3/planner/systemobjects.go +++ b/sql3/planner/systemobjects.go @@ -6,11 +6,11 @@ import ( "context" "time" - pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/dax" - "github.com/molecula/featurebase/v3/sql3" - "github.com/molecula/featurebase/v3/sql3/parser" - "github.com/molecula/featurebase/v3/sql3/planner/types" + pilosa "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/sql3" + "github.com/featurebasedb/featurebase/v3/sql3/parser" + "github.com/featurebasedb/featurebase/v3/sql3/planner/types" ) type viewSystemObject struct { From 965ad15829ddf3183bb4f9fdf7d245264577ee45 Mon Sep 17 00:00:00 2001 From: HHans09 <91560812+HHans09@users.noreply.github.com> Date: Tue, 17 Jan 2023 16:58:55 -0500 Subject: [PATCH 06/15] Fb 1876 : Implement Replicate() func (#2410) * fb-1876 : creating Replicate fun * fb-1876: creating string replicate func * Fb-1876: String Replicate func * Fb-1876: String Replicate func (cherry picked from commit 2fccb87dcfc575b7618e66c52f8304edae2754c7) --- sql3/planner/expression.go | 2 + sql3/planner/expressionanalyzercall.go | 6 ++- sql3/planner/inbuiltfunctionsstring.go | 34 +++++++++++++++- sql3/test/defs/defs_string_functions.go | 54 +++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 3 deletions(-) diff --git a/sql3/planner/expression.go b/sql3/planner/expression.go index 2253719a0..01926400c 100644 --- a/sql3/planner/expression.go +++ b/sql3/planner/expression.go @@ -1518,6 +1518,8 @@ func (n *callPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er return n.EvaluateSpace(currentRow) case "LEN": return n.EvaluateLen(currentRow) + case "REPLICATE": + return n.EvaluateReplicate(currentRow) default: return nil, sql3.NewErrInternalf("unhandled function name '%s'", n.name) } diff --git a/sql3/planner/expressionanalyzercall.go b/sql3/planner/expressionanalyzercall.go index 1b743b913..6dce440f2 100644 --- a/sql3/planner/expressionanalyzercall.go +++ b/sql3/planner/expressionanalyzercall.go @@ -264,13 +264,15 @@ func (p *ExecutionPlanner) analyzeCallExpression(call *parser.Call, scope parser case "LTRIM": return p.analyseFunctionTrim(call, scope) case "SUFFIX": - return p.analyseFunctionPrefixSuffix(call, scope) + return p.analyseFunctionPrefixSuffixReplicate(call, scope) case "PREFIX": - return p.analyseFunctionPrefixSuffix(call, scope) + return p.analyseFunctionPrefixSuffixReplicate(call, scope) case "SPACE": return p.analyseFunctionSpace(call, scope) case "LEN": return p.analyseFunctionLen(call, scope) + case "REPLICATE": + return p.analyseFunctionPrefixSuffixReplicate(call, scope) default: return nil, sql3.NewErrCallUnknownFunction(call.Name.NamePos.Line, call.Name.NamePos.Column, call.Name.Name) } diff --git a/sql3/planner/inbuiltfunctionsstring.go b/sql3/planner/inbuiltfunctionsstring.go index 52bb71e28..ccc31b90c 100644 --- a/sql3/planner/inbuiltfunctionsstring.go +++ b/sql3/planner/inbuiltfunctionsstring.go @@ -163,7 +163,7 @@ func (p *ExecutionPlanner) analyseFunctionTrim(call *parser.Call, scope parser.S return call, nil } -func (p *ExecutionPlanner) analyseFunctionPrefixSuffix(call *parser.Call, scope parser.Statement) (parser.Expr, error) { +func (p *ExecutionPlanner) analyseFunctionPrefixSuffixReplicate(call *parser.Call, scope parser.Statement) (parser.Expr, error) { if len(call.Args) != 2 { return nil, sql3.NewErrCallParameterCountMismatch(call.Rparen.Line, call.Rparen.Column, call.Name.Name, 2, len(call.Args)) } @@ -612,3 +612,35 @@ func (n *callPlanExpression) EvaluateLen(currentRow []interface{}) (interface{}, return int64(len([]rune(stringArg))), nil } +func (n *callPlanExpression) EvaluateReplicate(currentRow []interface{}) (interface{}, error) { + argEval, err := n.args[0].Evaluate(currentRow) + if err != nil { + return nil, err + } + if argEval == nil { + return nil, nil + } + stringArg, ok := argEval.(string) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type converion %T", argEval) + } + + argEval, err = n.args[1].Evaluate(currentRow) + if err != nil { + return nil, err + } + if argEval == nil { + return nil, nil + } + intArg, ok := argEval.(int64) + if !ok { + return nil, sql3.NewErrInternalf("unexpected type converion %T", argEval) + } + + if intArg < 0 { + return nil, sql3.NewErrValueOutOfRange(0, 0, intArg) + } + + return strings.Repeat(stringArg, int(intArg)), nil + +} diff --git a/sql3/test/defs/defs_string_functions.go b/sql3/test/defs/defs_string_functions.go index 54529285f..916844b9a 100644 --- a/sql3/test/defs/defs_string_functions.go +++ b/sql3/test/defs/defs_string_functions.go @@ -766,5 +766,59 @@ var stringScalarFunctionsTests = TableTest{ ), Compare: CompareExactOrdered, }, + { + name: "ReplicateString", + SQLs: sqls( + "select replicate('this',2)", + ), + ExpHdrs: hdrs( + hdr("", fldTypeString), + ), + ExpRows: rows( + row(string("thisthis")), + ), + Compare: CompareExactOrdered, + }, + { + name: "ReplicateNull", + SQLs: sqls( + "select replicate(null,null)", + ), + ExpHdrs: hdrs( + hdr("", fldTypeString), + ), + ExpRows: rows( + row(nil), + ), + Compare: CompareExactOrdered, + }, + { + name: "ReplicateincorrectArguments", + SQLs: sqls( + "select replicate('this',)", + ), + ExpErr: "[1:25] 'replicate': count of formal parameters (2) does not match count of actual parameters (1)", + }, + { + name: "ReplicateincorrectTypeofArguments", + SQLs: sqls( + "select replicate(1,2)", + ), + ExpErr: "[1:18] string expression expected", + }, + { + name: "ReplicateincorrectTypeofArguments", + SQLs: sqls( + "select replicate('this','this')", + ), + ExpErr: "[1:25] integer expression expected", + }, + { + name: "ReplicateOutofRange", + SQLs: sqls( + "select replicate('this',-1)", + ), + ExpErr: "[0:0] value '-1' out of range", + }, }, } From f2812309ac3de6d47603afbb24ccf095c7d549df Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula <85502298+pokeeffe-molecula@users.noreply.github.com> Date: Wed, 18 Jan 2023 10:35:14 -0600 Subject: [PATCH 07/15] make the count....count (#2416) (cherry picked from commit b6e642338a60241a7fb4168b7508a8db27138beb) --- sql3/planner/opinsert.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql3/planner/opinsert.go b/sql3/planner/opinsert.go index 626c1dbce..cb10283e8 100644 --- a/sql3/planner/opinsert.go +++ b/sql3/planner/opinsert.go @@ -364,12 +364,14 @@ func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) { } } + count := batch.Len() + if err := batch.Import(); err != nil { return nil, errors.Wrap(err, "importing batch") } // update the counter for inserts - pilosa.PerfCounterSQLInsertsSec.Add(int64(batch.Len())) + pilosa.PerfCounterSQLInsertsSec.Add(int64(count)) return nil, types.ErrNoMoreRows } From f6a767befb6cc55a788b35ed1aac1d65884d9bad Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 18 Jan 2023 14:27:45 -0600 Subject: [PATCH 08/15] Refactor CLI (#2417) This commit moves the cli out of the `ctl` package and into its own `cli` package. It also adds some basic tests for expected input. Finally, it fixes a bug which was causing intentional line feeds to be ignored, which was a problem with the BULK INSERT command. (cherry picked from commit cf72bfa16f3b750420ab66a08be6d56ef862dd27) --- {ctl => cli}/cli.go | 218 +++++++++++++---------------- cli/cli_test.go | 199 ++++++++++++++++++++++++++ {fbcloud => cli/fbcloud}/auth.go | 0 {fbcloud => cli/fbcloud}/client.go | 0 cli/queryer.go | 96 +++++++++++++ cmd/cli.go | 30 ++-- 6 files changed, 407 insertions(+), 136 deletions(-) rename {ctl => cli}/cli.go (72%) create mode 100644 cli/cli_test.go rename {fbcloud => cli/fbcloud}/auth.go (100%) rename {fbcloud => cli/fbcloud}/client.go (100%) create mode 100644 cli/queryer.go diff --git a/ctl/cli.go b/cli/cli.go similarity index 72% rename from ctl/cli.go rename to cli/cli.go index 77bafbd11..b839bf9b2 100644 --- a/ctl/cli.go +++ b/cli/cli.go @@ -1,9 +1,7 @@ -package ctl +package cli import ( - "bytes" "context" - "encoding/json" "fmt" "io" "net/http" @@ -32,6 +30,12 @@ const ( nullValue string = "NULL" ) +var ( + Stdin io.ReadCloser = os.Stdin + Stdout io.Writer = os.Stdout + Stderr io.Writer = os.Stderr +) + var ( splash string = fmt.Sprintf(`FeatureBase CLI (%s) Type "exit" to quit. @@ -55,35 +59,52 @@ type CLICommand struct { OrganizationID string `json:"org-id"` DatabaseID string `json:"db-id"` - queryer FBQueryer + Queryer Queryer `json:"-"` + + Stdin io.ReadCloser `json:"-"` + Stdout io.Writer `json:"-"` + Stderr io.Writer `json:"-"` } func NewCLICommand(logdest logger.Logger) *CLICommand { + return &CLICommand{ + Host: defaultHost, + HistoryPath: "", + + OrganizationID: "", + DatabaseID: "", + + Stdin: Stdin, + Stdout: Stdout, + Stderr: Stderr, + } +} + +func (cmd *CLICommand) setupHistory() { + // If HistoryPath has already been configured (i.e. with a command flag), + // don't bother setting up the default in the home directory. + if cmd.HistoryPath != "" { + return + } + historyPath := "" - home, err := os.UserHomeDir() - if err != nil { - fmt.Printf("Error getting home directory, command history persistence will be disabled: %v\n", err) + if home, err := os.UserHomeDir(); err != nil { + cmd.Printf("Error getting home directory, command history persistence will be disabled: %v\n", err) } else { historyDir := filepath.Join(home, ".featurebase") - err := os.MkdirAll(historyDir, 0750) + err := os.MkdirAll(historyDir, 0o750) if err != nil { - fmt.Printf("Creating directory for history: %v\n", err) + cmd.Printf("Creating directory for history: %v\n", err) } else { historyPath = filepath.Join(historyDir, "cli_history") } } - return &CLICommand{ - Host: defaultHost, - HistoryPath: historyPath, - - OrganizationID: "", - DatabaseID: "", - } + cmd.HistoryPath = historyPath } // printQualifiers displays the currently set OrganizationID and DatabaseID. func (cmd *CLICommand) printQualifiers() { - fmt.Printf(" Host: %s\n Org: %s\n DB: %s\n", + cmd.Printf(" Host: %s\n Org: %s\n DB: %s\n", hostPort(cmd.Host, cmd.Port), cmd.OrganizationID, cmd.DatabaseID, @@ -91,6 +112,12 @@ func (cmd *CLICommand) printQualifiers() { } func (cmd *CLICommand) setupClient() error { + // If the Queryer has already been set (in tests for example), don't bother + // trying to detect it. + if cmd.Queryer != nil { + return nil + } + if strings.TrimSpace(cmd.Host) == "" { return errors.Errorf("no host provided") } @@ -106,20 +133,20 @@ func (cmd *CLICommand) setupClient() error { switch typ { case featurebaseTypeStandard: - fmt.Println("Detected standard deployment") - cmd.queryer = &standardQueryer{ + cmd.Printf("Detected standard deployment\n") + cmd.Queryer = &standardQueryer{ Host: cmd.Host, Port: cmd.Port, } case featurebaseTypeDAX: - fmt.Println("Detected dax deployment") - cmd.queryer = &daxQueryer{ + cmd.Printf("Detected dax deployment\n") + cmd.Queryer = &daxQueryer{ Host: cmd.Host, Port: cmd.Port, } case featurebaseTypeCloud: - fmt.Println("Detected cloud deployment") - cmd.queryer = &fbcloud.Queryer{ + cmd.Printf("Detected cloud deployment\n") + cmd.Queryer = &fbcloud.Queryer{ Host: hostPort(cmd.Host, cmd.Port), ClientID: cmd.ClientID, @@ -222,9 +249,9 @@ func (cmd *CLICommand) detectFBType() (featurebaseType, error) { func (cmd *CLICommand) Run(ctx context.Context) error { // Print the splash message. - fmt.Print(splash) - err := cmd.setupClient() - if err != nil { + cmd.Printf(splash) + cmd.setupHistory() + if err := cmd.setupClient(); err != nil { return errors.Wrap(err, "setting up client") } cmd.printQualifiers() @@ -234,6 +261,10 @@ func (cmd *CLICommand) Run(ctx context.Context) error { HistoryFile: cmd.HistoryPath, HistoryLimit: 100000, DisableAutoSaveHistory: true, + + Stdin: cmd.Stdin, + Stdout: cmd.Stdout, + Stderr: cmd.Stderr, }) if err != nil { return errors.Wrap(err, "getting readline") @@ -254,7 +285,7 @@ func (cmd *CLICommand) Run(ctx context.Context) error { } else { rl.SetPrompt(promptBegin) // Add some white space before each new prompt. - fmt.Println() + cmd.Printf("\n") } // Read user provided input. @@ -263,6 +294,27 @@ func (cmd *CLICommand) Run(ctx context.Context) error { return errors.Wrap(err, "reading line") } + // We append a line feed at the end of each line because at this point + // we have effectively stripped any intentional line feeds (since we are + // reading a line at a time), and we don't want to do that. An example + // of an intentional line feed is in a BULK INSERT CSV STREAM like this + // example: + // + // bulk replace + // into foo (_id, age) + // map (0 id, 1 int) + // from + // x'3,33 + // 4,44 + // 5,55' + // with + // format 'CSV' + // input 'STREAM'; + // + // We want to preserve the line feeds that are contained in the x'' + // block; those are intentional as they demarc records within the csv. + line += "\n" + if !inMidCommand { // Handle the exit command. if line == exitCommand || line == exitCommand+terminationChar { @@ -284,7 +336,7 @@ func (cmd *CLICommand) Run(ctx context.Context) error { for i, part := range parts { partIsFinal := i == len(parts)-1 - partIsBlank := part == "" + partIsBlank := strings.TrimSpace(part) == "" if partIsBlank && partIsFinal { continue @@ -315,7 +367,7 @@ func (cmd *CLICommand) Run(ctx context.Context) error { err = rl.SaveHistory(strings.Join(cmd.commands, "; ") + ";") if err != nil { - fmt.Printf("Couldn't save history: %v\n", err) + cmd.Printf("Couldn't save history: %v\n", err) } if err := cmd.executeCommands(ctx); err != nil { @@ -330,14 +382,10 @@ func appendCommand(orig string, part string) string { if orig == "" { return part } else { - return orig + " " + part + return orig + part } } -type FBQueryer interface { - Query(org, db, sql string) (*featurebase.WireQueryResponse, error) -} - func (cmd *CLICommand) executeCommands(ctx context.Context) error { // Clear out the buffered commands on any exit from this method. defer func() { @@ -352,12 +400,12 @@ func (cmd *CLICommand) executeCommands(ctx context.Context) error { continue } - sqlResponse, err := cmd.queryer.Query(cmd.OrganizationID, cmd.DatabaseID, sql) + sqlResponse, err := cmd.Queryer.Query(cmd.OrganizationID, cmd.DatabaseID, sql) if err != nil { - fmt.Printf("making query: %v\n", err) + cmd.Printf("making query: %v\n", err) continue } - err = writeOut(sqlResponse, os.Stdout) + err = writeOut(sqlResponse, cmd.Stdout, cmd.Stderr) if err != nil { return errors.Wrap(err, "writing out response") } @@ -366,6 +414,12 @@ func (cmd *CLICommand) executeCommands(ctx context.Context) error { return nil } +// Printf is a helper method which sends the given payload to stdout. +func (cmd *CLICommand) Printf(format string, a ...any) { + out := fmt.Sprintf(format, a...) + cmd.Stdout.Write([]byte(out)) +} + // handleIfNonSQLCommand will handle special case command like "SET ..." and // "USE ...". If the sql command matches one of these conditions and is handled, // the bool returned will be true; @@ -431,19 +485,19 @@ func writeWarnings(r *featurebase.WireQueryResponse, w io.Writer) error { return nil } -func writeOut(r *featurebase.WireQueryResponse, w io.Writer) error { +func writeOut(r *featurebase.WireQueryResponse, wOut io.Writer, wErr io.Writer) error { if r == nil { return errors.New("attempt to write out nil response") } if r.Error != "" { - if _, err := w.Write([]byte("Error: " + r.Error + "\n")); err != nil { + if _, err := wErr.Write([]byte("Error: " + r.Error + "\n")); err != nil { return errors.Wrapf(err, "writing error: %s", r.Error) } - return writeWarnings(r, w) + return writeWarnings(r, wOut) } t := table.NewWriter() - t.SetOutputMirror(w) + t.SetOutputMirror(wOut) // Don't uppercase the header values. t.Style().Format.Header = text.FormatDefault @@ -461,7 +515,7 @@ func writeOut(r *featurebase.WireQueryResponse, w io.Writer) error { } t.Render() - err := writeWarnings(r, w) + err := writeWarnings(r, wOut) if err != nil { return err } @@ -474,7 +528,7 @@ func writeOut(r *featurebase.WireQueryResponse, w io.Writer) error { lifeAffirmingMessage = " (Sorry! That took longer than expected 😭)" } - if _, err := w.Write([]byte(fmt.Sprintf("\nExecution time: %dμs%s\n", r.ExecutionTime, lifeAffirmingMessage))); err != nil { + if _, err := wOut.Write([]byte(fmt.Sprintf("\nExecution time: %dμs%s\n", r.ExecutionTime, lifeAffirmingMessage))); err != nil { return errors.Wrapf(err, "writing execution time: %s", r.Error) } @@ -488,81 +542,3 @@ func schemaToRow(schema featurebase.WireQuerySchema) []interface{} { } return ret } - -// Ensure type implements interface. -var _ FBQueryer = (*standardQueryer)(nil) - -// standardQueryer supports a standard featurebase deployment hitting the /sql -// endpoint with a payload containing only the sql statement. -type standardQueryer struct { - Host string - Port string -} - -func (qryr *standardQueryer) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) { - buf := bytes.Buffer{} - url := fmt.Sprintf("%s/sql", hostPort(qryr.Host, qryr.Port)) - - buf.Write([]byte(sql)) - - resp, err := http.Post(url, "application/json", &buf) - if err != nil { - return nil, errors.Wrapf(err, "posting query") - } - - fullbod, err := io.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrap(err, "reading response") - } - sqlResponse := &featurebase.WireQueryResponse{} - // TODO(tlt): switch this back once all responses are typed - // if err := json.Unmarshal(fullbod, sqlResponse); err != nil { - if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil { - return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod) - } - - return sqlResponse, nil -} - -// Ensure type implements interface. -var _ FBQueryer = (*daxQueryer)(nil) - -// daxQueryer is similar to the standardQueryer except that it hits a different -// endpoint, and its payload is a json object which includes, in addition to the -// sql statement, things like org and db. -type daxQueryer struct { - Host string - Port string -} - -func (qryr *daxQueryer) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) { - buf := bytes.Buffer{} - url := fmt.Sprintf("%s/queryer/sql", hostPort(qryr.Host, qryr.Port)) - - sqlReq := &queryerhttp.SQLRequest{ - OrganizationID: dax.OrganizationID(org), - DatabaseID: dax.DatabaseID(db), - SQL: sql, - } - if err := json.NewEncoder(&buf).Encode(sqlReq); err != nil { - return nil, errors.Wrapf(err, "encoding sql request: %s", sql) - } - - resp, err := http.Post(url, "application/json", &buf) - if err != nil { - return nil, errors.Wrapf(err, "posting query") - } - - fullbod, err := io.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrap(err, "reading response") - } - sqlResponse := &featurebase.WireQueryResponse{} - // TODO(tlt): switch this back once all responses are typed - // if err := json.Unmarshal(fullbod, sqlResponse); err != nil { - if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil { - return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod) - } - - return sqlResponse, nil -} diff --git a/cli/cli_test.go b/cli/cli_test.go new file mode 100644 index 000000000..69bcb690f --- /dev/null +++ b/cli/cli_test.go @@ -0,0 +1,199 @@ +package cli_test + +import ( + "context" + "io" + "strings" + "sync" + "testing" + "time" + + featurebase "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/cli" + "github.com/molecula/featurebase/v3/logger" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func TestCLI(t *testing.T) { + t.Run("Input", func(t *testing.T) { + ctx := context.Background() + + capture := newCapture(t) + + cli := cli.NewCLICommand(logger.StderrLogger) + cli.Stdin = capture + cli.Stdout = capture + cli.Queryer = capture + + go func() { + assert.NoError(t, cli.Run(ctx)) + }() + + none := []string{} + + // One statement, one line. + capture.Assert("one;", []string{`one`}) + + // One statement, multiple lines. + capture.Assert("one", none) + capture.Assert(" two ", none) + capture.Assert("three;", []string{`one + two +three`}) + + // Multiple statements, one line. + capture.Assert("foo; bar;", []string{`foo`, `bar`}) + + // Multiple statements, multiple lines. + capture.Assert("a1", none) + capture.Assert("a2; b1", []string{`a1 +a2`}) + capture.Assert("b2;", []string{`b1 +b2`}) + + // Blank lines. + capture.Assert("one", none) + capture.Assert("", none) + capture.Assert("three;", []string{`one + +three`}) + + // Just a semi-colon. + capture.Assert(";", none) + + // Multi-line with just a semi-colon. + capture.Assert("one", none) + capture.Assert(";", []string{`one`}) + + // Ensure a clean exit with no errors. + assert.NoError(t, capture.Exit()) + }) +} + +//////////////////////////////////////////////////////// + +// Ensure type implementes interface. +var _ io.ReadCloser = (*capture)(nil) +var _ io.Writer = (*capture)(nil) +var _ cli.Queryer = (*capture)(nil) + +// capture implements the various CLI interfaces in order to capture test input +// and submit it as though that input were being read from the command line. It +// also captures calls made to the Queryer.Query method and ensures the sql the +// contain is expected. +type capture struct { + t *testing.T + + // ch is a channel of strings (one line at a time) of CLI input. + ch chan string + + mu sync.RWMutex + sqls []string + + // queryDone will receive an event any time the Query method is called and + // has completed. This is to tell the Assert method that it's safe to + // compare the sqls slice. + queryDone chan struct{} + + asserting chan struct{} + + err error +} + +func newCapture(t *testing.T) *capture { + return &capture{ + t: t, + ch: make(chan string), + sqls: make([]string, 0), + queryDone: make(chan struct{}), + } +} + +func (c *capture) Exit() error { + c.sendLine("exit") + c.mu.RLock() + defer c.mu.RUnlock() + return c.err +} + +func (c *capture) Assert(in string, out []string) { + c.asserting = make(chan struct{}) + + c.sendLine(in) + + // Wait for the CLI command to complete processing the input and send the + // sql to Query() by blocking on the queryDone channel. Because Query gets + // called for every sql statement in the input, an input resulting in + // multiple sql statements needs to wait for all expected queries to + // complete. A timeout is included to this so it doesn't deadlock in the + // case where Query is expected to be called, but isn't; after the timeout, + // the test should fail completely. In summary: we wait on queryDone the + // number of sql statements we expect. If we receive fewer than expected, + // the timeout will occur. If we receive more than expected, the Query() + // method will effectively deadlock, reach its own timout, then write to + // capture.err, which will be reported upon Exit(). + for range out { + select { + case <-c.queryDone: + case <-time.After(2 * time.Second): + c.t.Fatalf("expected Query() to be called") + } + } + + close(c.asserting) + + c.mu.Lock() + defer c.mu.Unlock() + + assert.Equal(c.t, out, c.sqls) + + // Reset the slice. + c.sqls = c.sqls[:0] + +} + +// sendLine sends the given string as a line input to the CLI command. It +// appends a line feed to the end of string in order to mimic the user hitting +// the return key. +func (c *capture) sendLine(s string) { + // Add a line feed before putting s on the channel in order to mimic the + // user hitting the return key. + c.ch <- s + "\n" +} + +// Read is read by the CLI in place of user input. It effectively sends lines of +// input to the CLI, getting each line to be sent off the channel. +func (c *capture) Read(b []byte) (n int, err error) { + s := <-c.ch + return strings.NewReader(s).Read(b) +} + +func (c *capture) Close() error { + close(c.ch) + return nil +} + +// Write is called with anything written to output. This would included results +// from calling Query() under normal, non-testing conditions, as well as other +// informational text sent to output, such as the splash message. +func (c *capture) Write(b []byte) (n int, err error) { + return 0, nil +} + +// Query is called by the CLI command once a full SQL statement is received +// (signified by the terminator: `;`). +func (c *capture) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) { + c.mu.Lock() + c.sqls = append(c.sqls, sql) + c.mu.Unlock() + + select { + case c.queryDone <- struct{}{}: + case <-c.asserting: + c.mu.Lock() + c.err = errors.Errorf("unexpected query: %s", sql) + c.mu.Unlock() + } + return &featurebase.WireQueryResponse{}, nil +} diff --git a/fbcloud/auth.go b/cli/fbcloud/auth.go similarity index 100% rename from fbcloud/auth.go rename to cli/fbcloud/auth.go diff --git a/fbcloud/client.go b/cli/fbcloud/client.go similarity index 100% rename from fbcloud/client.go rename to cli/fbcloud/client.go diff --git a/cli/queryer.go b/cli/queryer.go new file mode 100644 index 000000000..cd54c57e4 --- /dev/null +++ b/cli/queryer.go @@ -0,0 +1,96 @@ +package cli + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + featurebase "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/dax" + queryerhttp "github.com/molecula/featurebase/v3/dax/queryer/http" + "github.com/pkg/errors" +) + +type Queryer interface { + Query(org, db, sql string) (*featurebase.WireQueryResponse, error) +} + +// Ensure type implements interface. +var _ Queryer = (*standardQueryer)(nil) + +// standardQueryer supports a standard featurebase deployment hitting the /sql +// endpoint with a payload containing only the sql statement. +type standardQueryer struct { + Host string + Port string +} + +func (qryr *standardQueryer) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) { + buf := bytes.Buffer{} + url := fmt.Sprintf("%s/sql", hostPort(qryr.Host, qryr.Port)) + + buf.Write([]byte(sql)) + + resp, err := http.Post(url, "application/json", &buf) + if err != nil { + return nil, errors.Wrapf(err, "posting query") + } + + fullbod, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "reading response") + } + sqlResponse := &featurebase.WireQueryResponse{} + // TODO(tlt): switch this back once all responses are typed + // if err := json.Unmarshal(fullbod, sqlResponse); err != nil { + if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil { + return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod) + } + + return sqlResponse, nil +} + +// Ensure type implements interface. +var _ Queryer = (*daxQueryer)(nil) + +// daxQueryer is similar to the standardQueryer except that it hits a different +// endpoint, and its payload is a json object which includes, in addition to the +// sql statement, things like org and db. +type daxQueryer struct { + Host string + Port string +} + +func (qryr *daxQueryer) Query(org, db, sql string) (*featurebase.WireQueryResponse, error) { + buf := bytes.Buffer{} + url := fmt.Sprintf("%s/queryer/sql", hostPort(qryr.Host, qryr.Port)) + + sqlReq := &queryerhttp.SQLRequest{ + OrganizationID: dax.OrganizationID(org), + DatabaseID: dax.DatabaseID(db), + SQL: sql, + } + if err := json.NewEncoder(&buf).Encode(sqlReq); err != nil { + return nil, errors.Wrapf(err, "encoding sql request: %s", sql) + } + + resp, err := http.Post(url, "application/json", &buf) + if err != nil { + return nil, errors.Wrapf(err, "posting query") + } + + fullbod, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "reading response") + } + sqlResponse := &featurebase.WireQueryResponse{} + // TODO(tlt): switch this back once all responses are typed + // if err := json.Unmarshal(fullbod, sqlResponse); err != nil { + if err := sqlResponse.UnmarshalJSONTyped(fullbod, true); err != nil { + return nil, errors.Wrapf(err, "unmarshaling query response, body:\n'%s'\n", fullbod) + } + + return sqlResponse, nil +} diff --git a/cmd/cli.go b/cmd/cli.go index 82c1f1e68..d37d8c100 100644 --- a/cmd/cli.go +++ b/cmd/cli.go @@ -7,29 +7,29 @@ import ( "github.com/spf13/cobra" ) -var cli *ctl.CLICommand +var cliCmd *cli.CLICommand // newCLICommand runs the FeatureBase CLI subcommand for ingesting bulk data. func newCLICommand(logdest logger.Logger) *cobra.Command { - cli = ctl.NewCLICommand(logdest) - cliCmd := &cobra.Command{ + cliCmd = cli.NewCLICommand(logdest) + cobraCmd := &cobra.Command{ Use: "cli", Short: "Query FB with SQL3 from the command line", Long: ``, - RunE: usageErrorWrapper(cli), + RunE: usageErrorWrapper(cliCmd), } - flags := cliCmd.Flags() - flags.StringVarP(&cli.Host, "host", "", cli.Host, "hostname of FeatureBase.") - flags.StringVarP(&cli.Port, "port", "", cli.Port, "port of FeatureBase.") - flags.StringVar(&cli.HistoryPath, "history-path", cli.HistoryPath, "path for history files.") - flags.StringVar(&cli.OrganizationID, "org-id", cli.OrganizationID, "OrganizationID.") - flags.StringVar(&cli.DatabaseID, "db-id", cli.DatabaseID, "DatabaseID.") + flags := cobraCmd.Flags() + flags.StringVarP(&cliCmd.Host, "host", "", cliCmd.Host, "hostname of FeatureBase.") + flags.StringVarP(&cliCmd.Port, "port", "", cliCmd.Port, "port of FeatureBase.") + flags.StringVar(&cliCmd.HistoryPath, "history-path", cliCmd.HistoryPath, "path for history files.") + flags.StringVar(&cliCmd.OrganizationID, "org-id", cliCmd.OrganizationID, "OrganizationID.") + flags.StringVar(&cliCmd.DatabaseID, "db-id", cliCmd.DatabaseID, "DatabaseID.") - flags.StringVar(&cli.ClientID, "client-id", cli.ClientID, "Cognito Client ID for FeatureBase Cloud access.") - flags.StringVar(&cli.Region, "region", cli.Region, "Cloud region for FeatureBase Cloud access (e.g. us-east-2).") - flags.StringVar(&cli.Email, "email", cli.Email, "Email address for FeatureBase Cloud access.") - flags.StringVar(&cli.Password, "password", cli.Password, "Password for FeatureBase Cloud access.") + flags.StringVar(&cliCmd.ClientID, "client-id", cliCmd.ClientID, "Cognito Client ID for FeatureBase Cloud access.") + flags.StringVar(&cliCmd.Region, "region", cliCmd.Region, "Cloud region for FeatureBase Cloud access (e.g. us-east-2).") + flags.StringVar(&cliCmd.Email, "email", cliCmd.Email, "Email address for FeatureBase Cloud access.") + flags.StringVar(&cliCmd.Password, "password", cliCmd.Password, "Password for FeatureBase Cloud access.") - return cliCmd + return cobraCmd } From 1046e28338df7a54988b673476c43c946cffbb10 Mon Sep 17 00:00:00 2001 From: Jacob Brinlee <66123601+jrbrinlee1@users.noreply.github.com> Date: Wed, 18 Jan 2023 17:42:19 -0600 Subject: [PATCH 09/15] SUP-288 (#2414) * adding kafka consumer config options (--kafka-max-poll-interval, --kafka-session-timeout, --kafka-group-instance-id, --kafka-socket-keepalive-enable, and --consumer-close-timeout) * wrapping consumer.Close() in timeout. Will wait consumer-close-timeout seconds before forcing consumer to exit * clean up logs (cherry picked from commit 17cdc58d800356741f47b505eaba4df397c984c8) --- batch/batch.go | 4 ++-- idk/common/configMap.go | 29 +++++++++++++++++++++++++++++ idk/ingest.go | 10 +++++++--- idk/kafka/cmd.go | 20 +++++++++++++++----- idk/kafka/source.go | 41 ++++++++++++++++++++++++++++++----------- 5 files changed, 83 insertions(+), 21 deletions(-) diff --git a/batch/batch.go b/batch/batch.go index 4b3a90bcc..e9e275774 100644 --- a/batch/batch.go +++ b/batch/batch.go @@ -1245,7 +1245,7 @@ func (b *Batch) doImport(frags, clearFrags fragments) error { } ferr := b.importer.ImportRoaringBitmap(ctx, b.tbl.ID, fld, shard, viewMap, false) - b.log.Debugf("imp-roar %s,shard:%d,views:%d %v", field, shard, len(clearViewMap), time.Since(starty)) + b.log.Debugf("imp-roar field: %s, shard:%d, views:%d %v", field, shard, len(clearViewMap), time.Since(starty)) return errors.Wrapf(ferr, "importing data for %s", field) }) } @@ -1703,7 +1703,7 @@ func (b *Batch) importValueData() error { start := time.Now() fld := featurebase.FieldInfoToField(field) err := b.importer.DoImport(ctx, b.tbl.ID, fld, shard, path, data) - b.log.Debugf("imp-vals %s,shard:%d,data:%d %v", field, shard, len(data), time.Since(start)) + b.log.Debugf("imp-vals field: %s, shard: %d, data: %d %v", field.Name, shard, len(data), time.Since(start)) return errors.Wrapf(err, "importing values for field = %s", field.Name) }) startIdx = i diff --git a/idk/common/configMap.go b/idk/common/configMap.go index bfdb0700e..006364296 100644 --- a/idk/common/configMap.go +++ b/idk/common/configMap.go @@ -44,6 +44,8 @@ func LaunchKafkaEventConfirmer(producer *confluent.Producer, finished *int32, it return doneChan } +// For a list of confluent consumer configuraiton options, go here: +// https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) { var err error configMap := &confluent.ConfigMap{} @@ -188,5 +190,32 @@ func SetupConfluent(m *idk.ConfluentCommand) (*confluent.ConfigMap, error) { } } + if m.KafkaGroupInstanceId != "" { + err = configMap.SetKey("group.instance.id", m.KafkaGroupInstanceId) + if err != nil { + return nil, err + } + } + + if m.KafkaMaxPollInterval != "" { + err = configMap.SetKey("max.poll.interval.ms", m.KafkaMaxPollInterval) + if err != nil { + return nil, err + } + } + + if m.KafkaSessionTimeout != "" { + err = configMap.SetKey("session.timeout.ms", m.KafkaSessionTimeout) + if err != nil { + return nil, err + } + } + + if m.KafkaSocketKeepaliveEnable != "" { + err = configMap.SetKey("socket.keepalive.enable", m.KafkaSocketKeepaliveEnable) + if err != nil { + return nil, err + } + } return configMap, nil } diff --git a/idk/ingest.go b/idk/ingest.go index f3cf02485..bf77905bd 100644 --- a/idk/ingest.go +++ b/idk/ingest.go @@ -183,9 +183,13 @@ type ConfluentCommand struct { KafkaSslEndpointIdentificationAlgorithm string `help:"The endpoint identification algorithm used by clients to validate server host name (ssl.endpoint.identification.algorithm) "` KafkaEnableSslCertificateVerification bool `help:"(enable.ssl.certificate.verification)"` KafkaSocketTimeoutMs int `help:"(socket.timeout.ms)"` + KafkaSocketKeepaliveEnable string `help:"The (socket.keepalive.enable) kafka consumer configuration"` - KafkaClientId string `help:"(client.id)"` - KafkaDebug string `help:"Kafka debug string (debug)"` + KafkaClientId string `help:"(client.id)"` + KafkaDebug string `help:"The (debug) kafka consumer configuration. A comma-separated list of debug contexts to enable. Detailed Consumer: consumer,cgrp,topic,fetch. Set to 'all' for most verbose option."` + KafkaMaxPollInterval string `help:"The (max.poll.interval.ms) kafka consumer configuration. The max time the consumer can go without polling the broker. Consumer exits after this timeout."` + KafkaSessionTimeout string `help:"The (session.timeout.ms) kafka consumer configuration. The max time the consumer can go without sending a heartbeat to the broker"` + KafkaGroupInstanceId string `help:"The (group.instance.id) kafka consumer configuration."` KafkaSaslUsername string `help:"SASL authentication username (sasl.username)"` KafkaSaslPassword string `help:"SASL authentication password (sasl.password)"` @@ -311,7 +315,7 @@ func (m *Main) runIngester(c int, l *msgCounter) error { err = source.Close() if err != nil { if m.log != nil { - m.log.Printf("error on close %v", err) + m.log.Errorf("Closing source: %v", err) } } }() diff --git a/idk/kafka/cmd.go b/idk/kafka/cmd.go index 6262eaa54..5e20ad76e 100644 --- a/idk/kafka/cmd.go +++ b/idk/kafka/cmd.go @@ -13,7 +13,8 @@ type Main struct { Group string `help:"Kafka group."` Topics []string `help:"Kafka topics to read from."` Timeout time.Duration `help:"Time to wait for more records from Kafka before flushing a batch. 0 to disable."` - SkipOld bool `short:"" help:"Skip to the most recent Kafka message rather than starting at the beginning."` + SkipOld bool `short:"" help:"False sets kafka consumer configuration auto.offset.reset to earliest, True sets it to latest."` + ConsumerCloseTimeout int `help:"The amount of time in seconds to wait for the consumer to close properly."` } func NewMain() (*Main, error) { @@ -22,13 +23,16 @@ func NewMain() (*Main, error) { ConfluentCommand: idk.ConfluentCommand{ KafkaBootstrapServers: []string{"localhost:9092"}, }, - Group: "defaultgroup", - Topics: []string{"defaulttopic"}, - Timeout: time.Second, + Group: "defaultgroup", + Topics: []string{"defaulttopic"}, + Timeout: time.Second, + ConsumerCloseTimeout: 30, } + m.SchemaRegistryURL = "http://" + defaultRegistryHost - m.OffsetMode = true m.Main.Namespace = "ingester_kafka" + //m.Main.OffsetMode = m.OffsetMode + m.OffsetMode = true m.NewSource = func() (idk.Source, error) { source := NewSource() source.KafkaBootstrapServers = m.KafkaBootstrapServers @@ -43,6 +47,12 @@ func NewMain() (*Main, error) { source.SchemaRegistryUsername = m.SchemaRegistryUsername source.SchemaRegistryPassword = m.SchemaRegistryPassword source.Verbose = m.Verbose + source.KafkaMaxPollInterval = m.KafkaMaxPollInterval + source.KafkaSessionTimeout = m.KafkaSessionTimeout + source.KafkaGroupInstanceId = m.KafkaGroupInstanceId + source.KafkaDebug = m.KafkaDebug + source.KafkaSocketKeepaliveEnable = m.KafkaSocketKeepaliveEnable + source.consumerCloseTimeout = m.ConsumerCloseTimeout if err := source.Open(); err != nil { return nil, errors.Wrap(err, "opening source") diff --git a/idk/kafka/source.go b/idk/kafka/source.go index 206322eb7..bcc519d20 100644 --- a/idk/kafka/source.go +++ b/idk/kafka/source.go @@ -33,14 +33,15 @@ import ( // achieve concurrency, create multiple Sources. type Source struct { idk.ConfluentCommand - Topics []string - Group string - Log logger.Logger - Timeout time.Duration - SkipOld bool - Verbose bool - schema Schema - TLS idk.TLSConfig + Topics []string + Group string + Log logger.Logger + Timeout time.Duration + SkipOld bool + Verbose bool + schema Schema + TLS idk.TLSConfig + consumerCloseTimeout int spoolBase uint64 spool []confluent.TopicPartition @@ -257,7 +258,9 @@ func (s *Source) CommitMessages(recs []confluent.TopicPartition) ([]confluent.To return s.client.CommitOffsets(recs) } -// Open initializes the kafka source. +// Open initializes the kafka source. (i.e. creating and configuring a consumer) +// The configuration options for the confluentinc/confluent-kafka-go/kafka +// libarary are: https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md func (s *Source) Open() error { cfg, err := common.SetupConfluent(&s.ConfluentCommand) if err != nil { @@ -439,10 +442,26 @@ func (c *Source) generator() { func (s *Source) Close() error { if s.client != nil { if s.opened { // only close opened sources + var err error + closedReturned := make(chan error, 1) + // send quit message to polling routine & wait for it to exit s.quit <- struct{}{} s.wg.Wait() - err := s.client.Close() - s.opened = false + s.Log.Debugf("Trying to close consumer %s...", s.client.String()) + go func() { + closedReturned <- s.client.Close() + }() + start := time.Now() + select { + case err = <-closedReturned: + if err == nil { + s.Log.Debugf("Successfully closed consumer %s!", s.client.String()) + s.opened = false + } + case <-time.After(time.Duration(s.consumerCloseTimeout * 1000 * 1000 * 1000)): + err = fmt.Errorf("Unable to properly close consumer %s after %f seconds", s.client.String(), time.Since(start).Seconds()) + } + return errors.Wrap(err, "closing kafka consumer") } } From b3a552e0d3fc64e3faa4b40c3cad400e42df2bf6 Mon Sep 17 00:00:00 2001 From: Pat Okeeffe <85502298+paddyjok@users.noreply.github.com> Date: Thu, 19 Jan 2023 10:05:34 -0600 Subject: [PATCH 10/15] implement left join (fb-1888) (#2420) * implement left join * fixed failing test (cherry picked from commit 7386f131591e97cda0eadfa926690eabd38292fe) --- sql3/errors.go | 12 +++++++ sql3/parser/ast.go | 22 ++++++------ sql3/parser/ast_test.go | 28 +++++---------- sql3/parser/parser.go | 24 ++++++++----- sql3/parser/parser_test.go | 54 +++++++++++------------------ sql3/parser/token.go | 6 ++-- sql3/planner/compileselect.go | 14 +++++++- sql3/planner/opnestedloops.go | 65 +++++++++++++++++++++++++---------- sql3/test/defs/defs_join.go | 37 +++++++++++++++++++- 9 files changed, 165 insertions(+), 97 deletions(-) diff --git a/sql3/errors.go b/sql3/errors.go index aab3a4218..6e069bd18 100644 --- a/sql3/errors.go +++ b/sql3/errors.go @@ -10,6 +10,7 @@ import ( const ( ErrInternal errors.Code = "ErrInternal" + ErrUnsupported errors.Code = "ErrUnsupported" ErrCacheKeyNotFound errors.Code = "ErrCacheKeyNotFound" ErrDuplicateColumn errors.Code = "ErrDuplicateColumn" @@ -172,6 +173,17 @@ func NewErrInternalf(format string, a ...interface{}) error { ) } +func NewErrUnsupported(line, col int, is bool, thing string) error { + msg := fmt.Sprintf("[%d:%d] %s are not supported", line, col, thing) + if is { + msg = fmt.Sprintf("[%d:%d] %s is not supported", line, col, thing) + } + return errors.New( + ErrUnknownIdentifier, + msg, + ) +} + func NewErrCacheKeyNotFound(key uint64) error { return errors.New( ErrCacheKeyNotFound, diff --git a/sql3/parser/ast.go b/sql3/parser/ast.go index c071164c1..b508e918f 100644 --- a/sql3/parser/ast.go +++ b/sql3/parser/ast.go @@ -3893,13 +3893,14 @@ func (c *JoinClause) SourceFromAlias(alias string) Source { } type JoinOperator struct { - Comma Pos // position of comma - Natural Pos // position of NATURAL keyword - Left Pos // position of LEFT keyword - Outer Pos // position of OUTER keyword - Inner Pos // position of INNER keyword - Cross Pos // position of CROSS keyword - Join Pos // position of JOIN keyword + Comma Pos // position of comma + Left Pos // position of LEFT keyword + Right Pos // position of RIGHT keyword + Full Pos // position of FULL keyword + Outer Pos // position of OUTER keyword + Inner Pos // position of INNER keyword + // Cross Pos // position of CROSS keyword // TODO(pok) - add cross back when we do it + Join Pos // position of JOIN keyword } // Clone returns a deep copy of op. @@ -3918,9 +3919,6 @@ func (op *JoinOperator) String() string { } var buf bytes.Buffer - if op.Natural.IsValid() { - buf.WriteString(" NATURAL") - } if op.Left.IsValid() { buf.WriteString(" LEFT") if op.Outer.IsValid() { @@ -3928,8 +3926,8 @@ func (op *JoinOperator) String() string { } } else if op.Inner.IsValid() { buf.WriteString(" INNER") - } else if op.Cross.IsValid() { - buf.WriteString(" CROSS") + // } else if op.Cross.IsValid() { + // buf.WriteString(" CROSS") } buf.WriteString(" JOIN ") diff --git a/sql3/parser/ast_test.go b/sql3/parser/ast_test.go index c326e055b..b963f2161 100644 --- a/sql3/parser/ast_test.go +++ b/sql3/parser/ast_test.go @@ -831,18 +831,6 @@ func TestSelectStatement_String(t *testing.T) { }, }, `SELECT * FROM x JOIN y ON TRUE`) - AssertStatementStringer(t, &parser.SelectStatement{ - Columns: []*parser.ResultColumn{{Star: pos(0)}}, - Source: &parser.JoinClause{ - X: &parser.QualifiedTableName{Name: &parser.Ident{Name: "x"}}, - Operator: &parser.JoinOperator{Natural: pos(0), Inner: pos(0)}, - Y: &parser.QualifiedTableName{Name: &parser.Ident{Name: "y"}}, - Constraint: &parser.UsingConstraint{ - Columns: []*parser.Ident{{Name: "a"}, {Name: "b"}}, - }, - }, - }, `SELECT * FROM x NATURAL INNER JOIN y USING (a, b)`) - AssertStatementStringer(t, &parser.SelectStatement{ Columns: []*parser.ResultColumn{{Star: pos(0)}}, Source: &parser.JoinClause{ @@ -861,14 +849,14 @@ func TestSelectStatement_String(t *testing.T) { }, }, `SELECT * FROM x LEFT OUTER JOIN y`) - AssertStatementStringer(t, &parser.SelectStatement{ - Columns: []*parser.ResultColumn{{Star: pos(0)}}, - Source: &parser.JoinClause{ - X: &parser.QualifiedTableName{Name: &parser.Ident{Name: "x"}}, - Operator: &parser.JoinOperator{Cross: pos(0)}, - Y: &parser.QualifiedTableName{Name: &parser.Ident{Name: "y"}}, - }, - }, `SELECT * FROM x CROSS JOIN y`) + // AssertStatementStringer(t, &parser.SelectStatement{ + // Columns: []*parser.ResultColumn{{Star: pos(0)}}, + // Source: &parser.JoinClause{ + // X: &parser.QualifiedTableName{Name: &parser.Ident{Name: "x"}}, + // Operator: &parser.JoinOperator{Cross: pos(0)}, + // Y: &parser.QualifiedTableName{Name: &parser.Ident{Name: "y"}}, + // }, + // }, `SELECT * FROM x CROSS JOIN y`) } func TestUpdateStatement_String(t *testing.T) { diff --git a/sql3/parser/parser.go b/sql3/parser/parser.go index af27ab6e0..8b69d807f 100644 --- a/sql3/parser/parser.go +++ b/sql3/parser/parser.go @@ -2274,7 +2274,7 @@ func (p *Parser) parseSource() (source Source, err error) { for { // Exit immediately if not part of a join operator. switch p.peek() { - case COMMA, NATURAL, LEFT, INNER, CROSS, JOIN: + case COMMA, LEFT, RIGHT, FULL, INNER /*CROSS, */, JOIN: default: return source, nil } @@ -2340,21 +2340,28 @@ func (p *Parser) parseJoinOperator() (*JoinOperator, error) { return &op, nil } - if p.peek() == NATURAL { - op.Natural, _, _ = p.scan() - } - - // Parse "LEFT", "LEFT OUTER", "INNER", or "CROSS" + // Parse "INNER", "LEFT [OUTER]", "RIGHT [OUTER]", "FULL [OUTER]", or "CROSS" switch p.peek() { case LEFT: op.Left, _, _ = p.scan() if p.peek() == OUTER { op.Outer, _, _ = p.scan() } + case RIGHT: + op.Right, _, _ = p.scan() + if p.peek() == OUTER { + op.Outer, _, _ = p.scan() + } + case FULL: + op.Full, _, _ = p.scan() + if p.peek() == OUTER { + op.Outer, _, _ = p.scan() + } case INNER: op.Inner, _, _ = p.scan() - case CROSS: - op.Cross, _, _ = p.scan() + + // case CROSS: + // op.Cross, _, _ = p.scan() } // Parse final JOIN. @@ -2362,7 +2369,6 @@ func (p *Parser) parseJoinOperator() (*JoinOperator, error) { return &op, p.errorExpected(p.pos, p.tok, "JOIN") } op.Join, _, _ = p.scan() - return &op, nil } diff --git a/sql3/parser/parser_test.go b/sql3/parser/parser_test.go index 748ddf0f4..67d9316ac 100644 --- a/sql3/parser/parser_test.go +++ b/sql3/parser/parser_test.go @@ -1879,22 +1879,6 @@ func TestParser_ParseStatement(t *testing.T) { }, }, }) - AssertParseStatement(t, `SELECT * FROM foo NATURAL JOIN bar`, &parser.SelectStatement{ - Select: pos(0), - Columns: []*parser.ResultColumn{ - {Star: pos(7)}, - }, - From: pos(9), - Source: &parser.JoinClause{ - X: &parser.QualifiedTableName{ - Name: &parser.Ident{NamePos: pos(14), Name: "foo"}, - }, - Operator: &parser.JoinOperator{Natural: pos(18), Join: pos(26)}, - Y: &parser.QualifiedTableName{ - Name: &parser.Ident{NamePos: pos(31), Name: "bar"}, - }, - }, - }) AssertParseStatement(t, `SELECT * FROM foo INNER JOIN bar ON true`, &parser.SelectStatement{ Select: pos(0), Columns: []*parser.ResultColumn{ @@ -1986,22 +1970,22 @@ func TestParser_ParseStatement(t *testing.T) { }, }, }) - AssertParseStatement(t, `SELECT * FROM foo CROSS JOIN bar`, &parser.SelectStatement{ - Select: pos(0), - Columns: []*parser.ResultColumn{ - {Star: pos(7)}, - }, - From: pos(9), - Source: &parser.JoinClause{ - X: &parser.QualifiedTableName{ - Name: &parser.Ident{NamePos: pos(14), Name: "foo"}, - }, - Operator: &parser.JoinOperator{Cross: pos(18), Join: pos(24)}, - Y: &parser.QualifiedTableName{ - Name: &parser.Ident{NamePos: pos(29), Name: "bar"}, - }, - }, - }) + // AssertParseStatement(t, `SELECT * FROM foo CROSS JOIN bar`, &parser.SelectStatement{ + // Select: pos(0), + // Columns: []*parser.ResultColumn{ + // {Star: pos(7)}, + // }, + // From: pos(9), + // Source: &parser.JoinClause{ + // X: &parser.QualifiedTableName{ + // Name: &parser.Ident{NamePos: pos(14), Name: "foo"}, + // }, + // Operator: &parser.JoinOperator{Cross: pos(18), Join: pos(24)}, + // Y: &parser.QualifiedTableName{ + // Name: &parser.Ident{NamePos: pos(29), Name: "bar"}, + // }, + // }, + // }) /*AssertParseStatement(t, `WITH cte (foo, bar) AS (SELECT baz), xxx AS (SELECT yyy) SELECT bat`, &parser.SelectStatement{ WithClause: &parser.WithClause{ @@ -2234,10 +2218,12 @@ func TestParser_ParseStatement(t *testing.T) { AssertParseStatementError(t, `SELECT foo FROM foo INDEXED BY`, `1:30: expected index name, found 'EOF'`)*/ /*AssertParseStatementError(t, `SELECT foo FROM foo NOT`, `1:23: expected INDEXED, found 'EOF'`)*/ AssertParseStatementError(t, `SELECT * FROM foo INNER`, `1:23: expected JOIN, found 'EOF'`) - AssertParseStatementError(t, `SELECT * FROM foo CROSS`, `1:23: expected JOIN, found 'EOF'`) - AssertParseStatementError(t, `SELECT * FROM foo NATURAL`, `1:25: expected JOIN, found 'EOF'`) AssertParseStatementError(t, `SELECT * FROM foo LEFT`, `1:22: expected JOIN, found 'EOF'`) AssertParseStatementError(t, `SELECT * FROM foo LEFT OUTER`, `1:28: expected JOIN, found 'EOF'`) + AssertParseStatementError(t, `SELECT * FROM foo RIGHT`, `1:23: expected JOIN, found 'EOF'`) + AssertParseStatementError(t, `SELECT * FROM foo RIGHT OUTER`, `1:29: expected JOIN, found 'EOF'`) + AssertParseStatementError(t, `SELECT * FROM foo FULL`, `1:22: expected JOIN, found 'EOF'`) + AssertParseStatementError(t, `SELECT * FROM foo FULL OUTER`, `1:28: expected JOIN, found 'EOF'`) AssertParseStatementError(t, `SELECT * FROM foo,`, `1:18: expected table name or left paren, found 'EOF'`) AssertParseStatementError(t, `SELECT * FROM foo JOIN bar ON`, `1:29: expected expression, found 'EOF'`) AssertParseStatementError(t, `SELECT * FROM foo JOIN bar USING`, `1:32: expected left paren, found 'EOF'`) diff --git a/sql3/parser/token.go b/sql3/parser/token.go index bc765a6a1..af6721666 100644 --- a/sql3/parser/token.go +++ b/sql3/parser/token.go @@ -139,6 +139,7 @@ const ( FOR FOREIGN FROM + FULL FUNCTION GLOB GROUP @@ -170,7 +171,6 @@ const ( MATCH MAX MIN - NATURAL NO NOT NOTBETWEEN @@ -208,6 +208,7 @@ const ( RESTRICT RETURNS RETURN + RIGHT ROLLBACK ROW ROWS @@ -362,6 +363,7 @@ var tokens = [...]string{ FOR: "FOR", FOREIGN: "FOREIGN", FROM: "FROM", + FULL: "FULL", FUNCTION: "FUNCTION", GLOB: "GLOB", GROUP: "GROUP", @@ -394,7 +396,6 @@ var tokens = [...]string{ MAX: "MAX", MIN: "MIN", NO: "NO", - NATURAL: "NATURAL", NOT: "NOT", NOTBETWEEN: "NOTBETWEEN", NOTEXISTS: "NOTEXISTS", @@ -431,6 +432,7 @@ var tokens = [...]string{ RESTRICT: "RESTRICT", RETURNS: "RETURNS", RETURN: "RETURN", + RIGHT: "RIGHT", ROLLBACK: "ROLLBACK", ROW: "ROW", ROWS: "ROWS", diff --git a/sql3/planner/compileselect.go b/sql3/planner/compileselect.go index 0a112bed1..5682d24b1 100644 --- a/sql3/planner/compileselect.go +++ b/sql3/planner/compileselect.go @@ -268,6 +268,17 @@ func (p *ExecutionPlanner) compileSource(scope *PlanOpQuery, source parser.Sourc case *parser.JoinClause: scope.AddWarning("🦖 here there be dragons! JOINS are experimental.") + // what sort of join is it? + jType := joinTypeInner + if sourceExpr.Operator.Left.IsValid() { + jType = joinTypeLeft + } else if sourceExpr.Operator.Right.IsValid() { + return nil, sql3.NewErrUnsupported(sourceExpr.Operator.Right.Line, sourceExpr.Operator.Right.Column, false, "RIGHT join types") + } else if sourceExpr.Operator.Full.IsValid() { + return nil, sql3.NewErrUnsupported(sourceExpr.Operator.Full.Line, sourceExpr.Operator.Full.Column, false, "FULL join types") + } + + // handle the join condition var joinCondition types.PlanExpression if sourceExpr.Constraint == nil { scope.AddWarning("⚠️ cartesian products are never a good idea - are you missing a join constraint?") @@ -285,6 +296,7 @@ func (p *ExecutionPlanner) compileSource(scope *PlanOpQuery, source parser.Sourc } } + // compile top and bottom child ops topOp, err := p.compileSource(scope, sourceExpr.X) if err != nil { return nil, err @@ -293,7 +305,7 @@ func (p *ExecutionPlanner) compileSource(scope *PlanOpQuery, source parser.Sourc if err != nil { return nil, err } - return NewPlanOpNestedLoops(topOp, bottomOp, joinCondition), nil + return NewPlanOpNestedLoops(topOp, bottomOp, jType, joinCondition), nil case *parser.QualifiedTableName: diff --git a/sql3/planner/opnestedloops.go b/sql3/planner/opnestedloops.go index 71df5894b..d4116baa5 100644 --- a/sql3/planner/opnestedloops.go +++ b/sql3/planner/opnestedloops.go @@ -16,14 +16,16 @@ type PlanOpNestedLoops struct { top types.PlanOperator bottom types.PlanOperator cond types.PlanExpression + jType joinType warnings []string } -func NewPlanOpNestedLoops(top, bottom types.PlanOperator, condition types.PlanExpression) *PlanOpNestedLoops { +func NewPlanOpNestedLoops(top, bottom types.PlanOperator, jType joinType, condition types.PlanExpression) *PlanOpNestedLoops { return &PlanOpNestedLoops{ top: top, bottom: bottom, cond: condition, + jType: jType, warnings: make([]string, 0), } } @@ -71,14 +73,14 @@ func (p *PlanOpNestedLoops) Iterator(ctx context.Context, row types.Row) (types. } rowWidth := len(row) + len(p.top.Schema()) + len(p.bottom.Schema()) - return newNestedLoopsIter(ctx, joinTypeInner, topIter, p.bottom, row, p.cond, rowWidth, row), nil + return newNestedLoopsIter(ctx, p.jType, topIter, p.bottom, row, p.cond, rowWidth, row), nil } func (p *PlanOpNestedLoops) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { if len(children) != 2 { return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) } - return NewPlanOpNestedLoops(children[0], children[1], p.cond), nil + return NewPlanOpNestedLoops(children[0], children[1], p.jType, p.cond), nil } func (p *PlanOpNestedLoops) Expressions() []types.PlanExpression { @@ -101,9 +103,10 @@ func (p *PlanOpNestedLoops) WithUpdatedExpressions(exprs ...types.PlanExpression type joinType byte const ( - joinTypeInner joinType = iota - joinTypeLeft - joinTypeRight + joinTypeInner joinType = iota // records that have matching values in both tables + joinTypeLeft // all records from the left table, and the matched records from the right table + joinTypeRight // all records from the right table, and the matched records from the left table + joinTypeFull // all records when there is a match in either left or right table ) type nestedLoopsIter struct { @@ -178,7 +181,7 @@ func (i *nestedLoopsIter) loadBottom(ctx context.Context) (row types.Row, err er return rightRow, nil } -func (i *nestedLoopsIter) buildRow(primary, secondary types.Row) types.Row { +func (i *nestedLoopsIter) buildRow(primary, secondary types.Row) (types.Row, error) { row := make(types.Row, i.rowSize) primary = primary[len(i.originalRow):] @@ -186,19 +189,23 @@ func (i *nestedLoopsIter) buildRow(primary, secondary types.Row) types.Row { var first, second types.Row var secondOffset int switch i.typ { - case joinTypeRight: - first = secondary - second = primary - secondOffset = len(row) - len(second) - default: + case joinTypeLeft: first = primary second = secondary secondOffset = len(first) + + case joinTypeInner: + first = primary + second = secondary + secondOffset = len(first) + + default: + return nil, sql3.NewErrInternalf("unsupported join type %d", i.typ) } copy(row, first) copy(row[secondOffset:], second) - return row + return row, nil } func conditionIsTrue(ctx context.Context, row types.Row, cond types.PlanExpression) (bool, error) { @@ -222,16 +229,38 @@ func (i *nestedLoopsIter) Next(ctx context.Context) (types.Row, error) { secondary, err := i.loadBottom(ctx) if err != nil { if err == types.ErrNoMoreRows { - if !i.foundMatch && (i.typ == joinTypeLeft || i.typ == joinTypeRight) { - row := i.buildRow(primary, nil) - return row, nil + // no more rows from secondary + switch i.typ { + case joinTypeInner: + continue + + case joinTypeLeft: + if !i.foundMatch { + row, err := i.buildRow(primary, nil) + if err != nil { + return nil, err + } + return row, nil + } + continue + + case joinTypeRight: + return nil, sql3.NewErrInternalf("unhandled join type %v", i.typ) + + case joinTypeFull: + return nil, sql3.NewErrInternalf("unhandled join type %v", i.typ) + + default: + return nil, sql3.NewErrInternalf("unhandled join type %v", i.typ) } - continue } return nil, err } - row := i.buildRow(primary, secondary) + row, err := i.buildRow(primary, secondary) + if err != nil { + return nil, err + } matches, err := conditionIsTrue(ctx, row, i.cond) if err != nil { return nil, err diff --git a/sql3/test/defs/defs_join.go b/sql3/test/defs/defs_join.go index d1269fb18..bd23e95de 100644 --- a/sql3/test/defs/defs_join.go +++ b/sql3/test/defs/defs_join.go @@ -17,6 +17,7 @@ var joinTestsUsers = TableTest{ srcRow(int64(1), string("b"), int64(18)), srcRow(int64(2), string("c"), int64(28)), srcRow(int64(3), string("d"), int64(34)), + srcRow(int64(4), string("e"), int64(36)), ), ), SQLTests: nil, @@ -44,7 +45,7 @@ var joinTestsOrders = TableTest{ } var joinTests = TableTest{ - name: "innerjointest", + name: "joinTests", SQLTests: []SQLTest{ { name: "innerjoin-aggregate-groupby", @@ -102,6 +103,40 @@ var joinTests = TableTest{ ), Compare: CompareExactOrdered, }, + { + name: "leftjoin", + SQLs: sqls( + "select u._id , o.userid from users u left join orders o on o.userid = u._id;", + ), + ExpHdrs: hdrs( + hdr("_id", fldTypeID), + hdr("userid", fldTypeInt), + ), + ExpRows: rows( + row(int64(0), int64(0)), + row(int64(1), int64(1)), + row(int64(1), int64(1)), + row(int64(2), int64(2)), + row(int64(2), int64(2)), + row(int64(3), int64(3)), + row(int64(4), nil), + ), + Compare: CompareExactOrdered, + }, + { + name: "fulljoin", + SQLs: sqls( + "select u._id , o.userid from users u full join orders o on o.userid = u._id;", + ), + ExpErr: "FULL join types are not supported", + }, + { + name: "outerjoin", + SQLs: sqls( + "select u._id , o.userid from users u right join orders o on o.userid = u._id;", + ), + ExpErr: "RIGHT join types are not supported", + }, }, PQLTests: []PQLTest{ { From 15eafa9825cf0d4b16d4325f127744c8fec66eb8 Mon Sep 17 00:00:00 2001 From: Fletcher Haynes Date: Thu, 19 Jan 2023 10:52:04 -0800 Subject: [PATCH 11/15] Add version checkin (#2413) * Initial commit of code to do a version check-in on startup * Add json tag to the response struct for version check * Adjusted version check response types * Changed error message in version check-in goroutine to use the logger. Changed URL to prod from dev. * Updated version checkin URL to be analytics Co-authored-by: Fletcher Haynes (cherry picked from commit c38210bef536c82cbbbf7b983147d51aaa194a57) --- server.go | 12 +++++++++++ verchk.go | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 verchk.go diff --git a/server.go b/server.go index 45efd6910..5bbff5f76 100644 --- a/server.go +++ b/server.go @@ -605,6 +605,18 @@ func (s *Server) Open() error { log.Println(errors.Wrap(err, "logging startup")) } + // Do version check in. This is in a goroutine so that we don't block server startup if the server endpoint is down/having issues. + go func() { + s.logger.Printf("Beginning featurebase version check-in") + vc := VersionChecker{URL: "https://analytics.featurebase.com/v2/featurebase/version"} + resp, err := vc.CheckIn() + if err != nil { + s.logger.Errorf("doing version checkin. Error was %s", err) + return + } + s.logger.Printf("Version check-in complete. Latest version is %s", resp.Information.Version) + }() + // Start DisCo. ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() diff --git a/verchk.go b/verchk.go new file mode 100644 index 000000000..f9af36e6a --- /dev/null +++ b/verchk.go @@ -0,0 +1,60 @@ +package pilosa + +import ( + "bytes" + "encoding/json" + "io" + "net/http" +) + +type VersionChecker struct { + URL string +} + +func NewVersionChecker(endpoint string) *VersionChecker { + v := VersionChecker{ + URL: endpoint, + } + return &v +} + +func (v *VersionChecker) CheckIn() (*Response, error) { + + body := make(map[string]string, 2) + body["entry_type"] = "user" + body["version"] = VersionInfo(true) + req, err := json.Marshal(body) + if err != nil { + return nil, err + } + _, err = http.Post(v.URL, "application/json", bytes.NewBuffer(req)) + if err != nil { + return nil, err + } + + var json_resp Response + r, err := http.Get(v.URL) + if err != nil { + return nil, err + } + data, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + + err = json.Unmarshal(data, &json_resp) + if err != nil { + + return nil, err + } + return &json_resp, nil + +} + +type Response struct { + Information InfoSubResponse `json:"info"` +} + +type InfoSubResponse struct { + Version string `json:"version"` +} From 82a398980d104acad0fed2f2b274a15bcc70dd0c Mon Sep 17 00:00:00 2001 From: Joe Friedrich Date: Thu, 19 Jan 2023 22:14:06 +0000 Subject: [PATCH 12/15] fix import sync "7f6ea0e6e..c38210bef" --- cli/queryer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/queryer.go b/cli/queryer.go index cd54c57e4..80f5b5fce 100644 --- a/cli/queryer.go +++ b/cli/queryer.go @@ -7,8 +7,8 @@ import ( "io" "net/http" - featurebase "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/dax" + featurebase "github.com/featurebasedb/featurebase/v3" + "github.com/molecula/featurebasedb/v3/dax" queryerhttp "github.com/molecula/featurebase/v3/dax/queryer/http" "github.com/pkg/errors" ) From c025daa2266b5ea6e0aeec96032dc18e05ba6eb9 Mon Sep 17 00:00:00 2001 From: Joe Friedrich Date: Fri, 20 Jan 2023 01:01:18 +0000 Subject: [PATCH 13/15] Fixed dep paths --- cli/cli.go | 2 +- cli/cli_test.go | 6 +++--- cli/queryer.go | 4 ++-- dax/mds/controller/balancer.go | 3 --- dax/mds/controller/controller_test.go | 1 - dax/mds/mds.go | 2 +- dax/mds/schemar/schemar.go | 2 -- go.mod | 8 ++++---- go.sum | 8 ++++++++ prometheus/prometheus_test.go | 3 ++- server/server.go | 1 - 11 files changed, 21 insertions(+), 19 deletions(-) diff --git a/cli/cli.go b/cli/cli.go index b839bf9b2..bc717624f 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -16,7 +16,7 @@ import ( featurebase "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/dax" queryerhttp "github.com/featurebasedb/featurebase/v3/dax/queryer/http" - "github.com/featurebasedb/featurebase/v3/fbcloud" + "github.com/featurebasedb/featurebase/v3/cli/fbcloud" "github.com/featurebasedb/featurebase/v3/logger" "github.com/pkg/errors" ) diff --git a/cli/cli_test.go b/cli/cli_test.go index 69bcb690f..ceac7a059 100644 --- a/cli/cli_test.go +++ b/cli/cli_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" - featurebase "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/cli" - "github.com/molecula/featurebase/v3/logger" + featurebase "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/cli" + "github.com/featurebasedb/featurebase/v3/logger" "github.com/pkg/errors" "github.com/stretchr/testify/assert" ) diff --git a/cli/queryer.go b/cli/queryer.go index 80f5b5fce..bde2b6cbc 100644 --- a/cli/queryer.go +++ b/cli/queryer.go @@ -8,8 +8,8 @@ import ( "net/http" featurebase "github.com/featurebasedb/featurebase/v3" - "github.com/molecula/featurebasedb/v3/dax" - queryerhttp "github.com/molecula/featurebase/v3/dax/queryer/http" + "github.com/featurebasedb/featurebase/v3/dax" + queryerhttp "github.com/featurebasedb/featurebase/v3/dax/queryer/http" "github.com/pkg/errors" ) diff --git a/dax/mds/controller/balancer.go b/dax/mds/controller/balancer.go index 567c4b8c7..2344af873 100644 --- a/dax/mds/controller/balancer.go +++ b/dax/mds/controller/balancer.go @@ -1,9 +1,6 @@ package controller import ( - "context" - "fmt" - "github.com/featurebasedb/featurebase/v3/dax" ) diff --git a/dax/mds/controller/controller_test.go b/dax/mds/controller/controller_test.go index d135738f5..9b204f55b 100644 --- a/dax/mds/controller/controller_test.go +++ b/dax/mds/controller/controller_test.go @@ -9,7 +9,6 @@ import ( "github.com/featurebasedb/featurebase/v3/dax" "github.com/featurebasedb/featurebase/v3/dax/mds/controller" - "github.com/featurebasedb/featurebase/v3/dax/mds/controller/naive/boltdb" daxtest "github.com/featurebasedb/featurebase/v3/dax/test" testbolt "github.com/featurebasedb/featurebase/v3/dax/test/boltdb" "github.com/featurebasedb/featurebase/v3/errors" diff --git a/dax/mds/mds.go b/dax/mds/mds.go index ffa64e441..657ee6461 100644 --- a/dax/mds/mds.go +++ b/dax/mds/mds.go @@ -11,7 +11,7 @@ import ( "github.com/featurebasedb/featurebase/v3/dax/boltdb" "github.com/featurebasedb/featurebase/v3/dax/computer" "github.com/featurebasedb/featurebase/v3/dax/mds/controller" - naiveboltdb "github.com/featurebasedb/featurebase/v3/dax/mds/controller/naive/boltdb" + balancerboltdb "github.com/featurebasedb/featurebase/v3/dax/mds/controller/balancer/boltdb" "github.com/featurebasedb/featurebase/v3/dax/mds/poller" "github.com/featurebasedb/featurebase/v3/dax/mds/schemar" schemarboltdb "github.com/featurebasedb/featurebase/v3/dax/mds/schemar/boltdb" diff --git a/dax/mds/schemar/schemar.go b/dax/mds/schemar/schemar.go index 1b45dce9f..7f54f35ee 100644 --- a/dax/mds/schemar/schemar.go +++ b/dax/mds/schemar/schemar.go @@ -2,8 +2,6 @@ package schemar import ( - "context" - "github.com/featurebasedb/featurebase/v3/dax" ) diff --git a/go.mod b/go.mod index 9710bf3c7..f92aa51ab 100644 --- a/go.mod +++ b/go.mod @@ -43,8 +43,8 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pelletier/go-toml v1.9.5 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.12.2 - github.com/prometheus/client_model v0.2.0 + github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_model v0.3.0 github.com/prometheus/prom2json v1.3.1 github.com/rakyll/statik v0.1.7 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect @@ -158,8 +158,8 @@ require ( github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/prometheus/common v0.33.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/sirupsen/logrus v1.7.0 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/afero v1.6.0 // indirect diff --git a/go.sum b/go.sum index 7657c22ad..b5324ea26 100644 --- a/go.sum +++ b/go.sum @@ -920,6 +920,8 @@ github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -927,6 +929,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -939,6 +943,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.33.0 h1:rHgav/0a6+uYgGdNt3jwz8FNSesO/Hsang3O0T9A5SE= github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -950,6 +956,8 @@ github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/prom2json v1.3.1 h1:OogL5hsrJpLPz3jZ4LPz4sJRTtADzViCNRQoqrzUQvk= github.com/prometheus/prom2json v1.3.1/go.mod h1:A8Oy9aiQx4wrJY9ya1i4nHOySGmkVp5EO0aU1iSJR+g= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= diff --git a/prometheus/prometheus_test.go b/prometheus/prometheus_test.go index 391124ba8..3e60c4af5 100644 --- a/prometheus/prometheus_test.go +++ b/prometheus/prometheus_test.go @@ -5,7 +5,8 @@ package prometheus_test import ( "testing" - pilosaPrometheus "github.com/featurebasedb/featurebase/v3/prometheus" + + "github.com/featurebasedb/featurebase/v3/test" "github.com/prometheus/client_golang/prometheus" io_prometheus_client "github.com/prometheus/client_model/go" ) diff --git a/server/server.go b/server/server.go index 72606c009..7fa2d2904 100644 --- a/server/server.go +++ b/server/server.go @@ -44,7 +44,6 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3/planner" "github.com/featurebasedb/featurebase/v3/statik" "github.com/featurebasedb/featurebase/v3/stats" - "github.com/featurebasedb/featurebase/v3/statsd" "github.com/featurebasedb/featurebase/v3/systemlayer" "github.com/featurebasedb/featurebase/v3/syswrap" "github.com/featurebasedb/featurebase/v3/testhook" From 9d095ca6c27e779b638ce6b9999a3a99676ec1a7 Mon Sep 17 00:00:00 2001 From: Joe Friedrich Date: Fri, 20 Jan 2023 01:06:54 +0000 Subject: [PATCH 14/15] Fixed controller import path --- dax/mds/controller/controller_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dax/mds/controller/controller_test.go b/dax/mds/controller/controller_test.go index 9b204f55b..fd8dea179 100644 --- a/dax/mds/controller/controller_test.go +++ b/dax/mds/controller/controller_test.go @@ -9,6 +9,8 @@ import ( "github.com/featurebasedb/featurebase/v3/dax" "github.com/featurebasedb/featurebase/v3/dax/mds/controller" + balancerdb "github.com/featurebasedb/featurebase/v3/dax/mds/controller/balancer/boltdb" + schemardb "github.com/featurebasedb/featurebase/v3/dax/mds/schemar/boltdb" daxtest "github.com/featurebasedb/featurebase/v3/dax/test" testbolt "github.com/featurebasedb/featurebase/v3/dax/test/boltdb" "github.com/featurebasedb/featurebase/v3/errors" From 7da67caa99a2cca1ea70292c14a7b94b38e1864a Mon Sep 17 00:00:00 2001 From: Joe Friedrich Date: Fri, 20 Jan 2023 02:11:00 +0000 Subject: [PATCH 15/15] fix go deps, add lattice --- api.go | 2 +- cache.go | 1 - cli/cli.go | 2 -- client/client.go | 1 - cmd/cli.go | 2 +- dax/queryer/orchestrator.go | 2 +- dax/queryer/queryer.go | 1 - field.go | 1 - fragment.go | 1 - holder.go | 1 - http_handler.go | 1 + idk/ingest.go | 2 -- index.go | 1 - lattice/src/services/__mocks__/eventServices.tsx | 1 + server.go | 1 - server/grpc.go | 1 - server/server.go | 2 -- sql3/planner/compilealtertable.go | 2 -- sql3/planner/compilebulkinsert.go | 5 ++++- sql3/planner/compiledroptable.go | 1 - sql3/planner/compileinsert.go | 1 - sql3/planner/executionplanner.go | 3 +++ sql3/planner/opbulkinsert.go | 1 + sql3/planner/oppqldistinctscan.go | 1 - sql3/planner/oppqltablescan.go | 1 - view.go | 1 - 26 files changed, 13 insertions(+), 26 deletions(-) diff --git a/api.go b/api.go index 5e7bfa418..9150cb14a 100644 --- a/api.go +++ b/api.go @@ -29,12 +29,12 @@ import ( "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/disco" "github.com/featurebasedb/featurebase/v3/rbf" + "github.com/prometheus/client_golang/prometheus" //"github.com/featurebasedb/featurebase/v3/pg" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" planner_types "github.com/featurebasedb/featurebase/v3/sql3/planner/types" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/featurebasedb/featurebase/v3/tracing" "github.com/pkg/errors" "golang.org/x/sync/errgroup" diff --git a/cache.go b/cache.go index 4577d69c9..156826180 100644 --- a/cache.go +++ b/cache.go @@ -13,7 +13,6 @@ import ( "github.com/featurebasedb/featurebase/v3/lru" pb "github.com/featurebasedb/featurebase/v3/proto" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/pkg/errors" ) diff --git a/cli/cli.go b/cli/cli.go index bc717624f..c9b4ca9da 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -14,8 +14,6 @@ import ( "github.com/jedib0t/go-pretty/table" "github.com/jedib0t/go-pretty/text" featurebase "github.com/featurebasedb/featurebase/v3" - "github.com/featurebasedb/featurebase/v3/dax" - queryerhttp "github.com/featurebasedb/featurebase/v3/dax/queryer/http" "github.com/featurebasedb/featurebase/v3/cli/fbcloud" "github.com/featurebasedb/featurebase/v3/logger" "github.com/pkg/errors" diff --git a/client/client.go b/client/client.go index 2dca46206..954cde9d7 100644 --- a/client/client.go +++ b/client/client.go @@ -29,7 +29,6 @@ import ( "github.com/featurebasedb/featurebase/v3/pb" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/featurebasedb/featurebase/v3/vprint" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" diff --git a/cmd/cli.go b/cmd/cli.go index d37d8c100..3a97ad47d 100644 --- a/cmd/cli.go +++ b/cmd/cli.go @@ -2,7 +2,7 @@ package cmd import ( - "github.com/featurebasedb/featurebase/v3/ctl" + "github.com/featurebasedb/featurebase/v3/cli" "github.com/featurebasedb/featurebase/v3/logger" "github.com/spf13/cobra" ) diff --git a/dax/queryer/orchestrator.go b/dax/queryer/orchestrator.go index 5d6ce48c9..75069b940 100644 --- a/dax/queryer/orchestrator.go +++ b/dax/queryer/orchestrator.go @@ -11,10 +11,10 @@ import ( featurebase "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/dax" + "github.com/prometheus/client_golang/prometheus" "github.com/featurebasedb/featurebase/v3/errors" "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/pql" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/featurebasedb/featurebase/v3/tracing" "golang.org/x/sync/errgroup" ) diff --git a/dax/queryer/queryer.go b/dax/queryer/queryer.go index 8b052d2db..ad2e5cd4e 100644 --- a/dax/queryer/queryer.go +++ b/dax/queryer/queryer.go @@ -22,7 +22,6 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner" plannertypes "github.com/featurebasedb/featurebase/v3/sql3/planner/types" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/featurebasedb/featurebase/v3/systemlayer" uuid "github.com/satori/go.uuid" ) diff --git a/field.go b/field.go index bd21f63ea..4647aaa5c 100644 --- a/field.go +++ b/field.go @@ -18,7 +18,6 @@ import ( "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/featurebasedb/featurebase/v3/testhook" "github.com/featurebasedb/featurebase/v3/tracing" "github.com/pkg/errors" diff --git a/fragment.go b/fragment.go index a8731d605..3ba4a2df9 100644 --- a/fragment.go +++ b/fragment.go @@ -24,7 +24,6 @@ import ( "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" "github.com/featurebasedb/featurebase/v3/shardwidth" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/featurebasedb/featurebase/v3/testhook" "github.com/featurebasedb/featurebase/v3/tracing" "github.com/featurebasedb/featurebase/v3/vprint" diff --git a/holder.go b/holder.go index ba079cc9b..b7e127750 100644 --- a/holder.go +++ b/holder.go @@ -18,7 +18,6 @@ import ( "github.com/featurebasedb/featurebase/v3/logger" rbfcfg "github.com/featurebasedb/featurebase/v3/rbf/cfg" "github.com/featurebasedb/featurebase/v3/roaring" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/featurebasedb/featurebase/v3/storage" "github.com/featurebasedb/featurebase/v3/testhook" "github.com/featurebasedb/featurebase/v3/vprint" diff --git a/http_handler.go b/http_handler.go index d775554c3..142daabe1 100644 --- a/http_handler.go +++ b/http_handler.go @@ -46,6 +46,7 @@ import ( "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/featurebasedb/featurebase/v3/wireprotocol" dto "github.com/prometheus/client_model/go" "github.com/prometheus/prom2json" uuid "github.com/satori/go.uuid" diff --git a/idk/ingest.go b/idk/ingest.go index bf77905bd..ab4a9b640 100644 --- a/idk/ingest.go +++ b/idk/ingest.go @@ -34,9 +34,7 @@ import ( "github.com/featurebasedb/featurebase/v3/idk/mds" "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/pql" - "github.com/featurebasedb/featurebase/v3/prometheus" proto "github.com/featurebasedb/featurebase/v3/proto" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/pkg/errors" prom "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" diff --git a/index.go b/index.go index 6914f7197..ae6d18e84 100644 --- a/index.go +++ b/index.go @@ -17,7 +17,6 @@ import ( "github.com/featurebasedb/featurebase/v3/disco" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/featurebasedb/featurebase/v3/testhook" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" diff --git a/lattice/src/services/__mocks__/eventServices.tsx b/lattice/src/services/__mocks__/eventServices.tsx index a6203b244..0fb72dece 100644 --- a/lattice/src/services/__mocks__/eventServices.tsx +++ b/lattice/src/services/__mocks__/eventServices.tsx @@ -10,3 +10,4 @@ const pilosa = { }; module.exports.pilosa = pilosa; +export {}; diff --git a/server.go b/server.go index 5bbff5f76..1a02fc5d1 100644 --- a/server.go +++ b/server.go @@ -27,7 +27,6 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" planner_types "github.com/featurebasedb/featurebase/v3/sql3/planner/types" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/featurebasedb/featurebase/v3/storage" "github.com/pkg/errors" "golang.org/x/sync/errgroup" diff --git a/server/grpc.go b/server/grpc.go index cecfa3bea..8090e60b3 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -22,7 +22,6 @@ import ( pb "github.com/featurebasedb/featurebase/v3/proto" vdsm_pb "github.com/featurebasedb/featurebase/v3/proto/vdsm" "github.com/featurebasedb/featurebase/v3/sql" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/improbable-eng/grpc-web/go/grpcweb" "github.com/pkg/errors" "google.golang.org/grpc" diff --git a/server/server.go b/server/server.go index 7fa2d2904..1af669555 100644 --- a/server/server.go +++ b/server/server.go @@ -39,11 +39,9 @@ import ( "github.com/featurebasedb/featurebase/v3/gopsutil" "github.com/featurebasedb/featurebase/v3/logger" pnet "github.com/featurebasedb/featurebase/v3/net" - "github.com/featurebasedb/featurebase/v3/prometheus" "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/planner" "github.com/featurebasedb/featurebase/v3/statik" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/featurebasedb/featurebase/v3/systemlayer" "github.com/featurebasedb/featurebase/v3/syswrap" "github.com/featurebasedb/featurebase/v3/testhook" diff --git a/sql3/planner/compilealtertable.go b/sql3/planner/compilealtertable.go index 30767a1f6..cff677f89 100644 --- a/sql3/planner/compilealtertable.go +++ b/sql3/planner/compilealtertable.go @@ -6,12 +6,10 @@ import ( "context" "strings" - pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/dax" "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" - "github.com/pkg/errors" ) type alterOperation int64 diff --git a/sql3/planner/compilebulkinsert.go b/sql3/planner/compilebulkinsert.go index 466df5040..ce80d024b 100644 --- a/sql3/planner/compilebulkinsert.go +++ b/sql3/planner/compilebulkinsert.go @@ -14,7 +14,10 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" - "github.com/pkg/errors" + goerrors "github.com/pkg/errors" + + + ) // compileBulkInsertStatement compiles a BULK INSERT statement into a diff --git a/sql3/planner/compiledroptable.go b/sql3/planner/compiledroptable.go index 2d70b8cfd..96b9f7ee1 100644 --- a/sql3/planner/compiledroptable.go +++ b/sql3/planner/compiledroptable.go @@ -10,7 +10,6 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" - "github.com/pkg/errors" ) // compileDropTableStatement compiles a DROP TABLE statement into a diff --git a/sql3/planner/compileinsert.go b/sql3/planner/compileinsert.go index 06eaf11af..4b6ae33dd 100644 --- a/sql3/planner/compileinsert.go +++ b/sql3/planner/compileinsert.go @@ -11,7 +11,6 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" - "github.com/pkg/errors" ) // compileInsertStatement compiles an INSERT statement into a PlanOperator. diff --git a/sql3/planner/executionplanner.go b/sql3/planner/executionplanner.go index 265203d5e..46657ee97 100644 --- a/sql3/planner/executionplanner.go +++ b/sql3/planner/executionplanner.go @@ -12,10 +12,13 @@ import ( "strings" pilosa "github.com/featurebasedb/featurebase/v3" + "github.com/featurebasedb/featurebase/v3/dax" + "github.com/featurebasedb/featurebase/v3/errors" "github.com/featurebasedb/featurebase/v3/logger" "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" + "golang.org/x/sync/errgroup" ) func isTableNotFoundError(err error) bool { diff --git a/sql3/planner/opbulkinsert.go b/sql3/planner/opbulkinsert.go index 01f905c85..294e65f0f 100644 --- a/sql3/planner/opbulkinsert.go +++ b/sql3/planner/opbulkinsert.go @@ -17,6 +17,7 @@ import ( "github.com/PaesslerAG/gval" "github.com/PaesslerAG/jsonpath" + pilosa "github.com/featurebasedb/featurebase/v3" "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" diff --git a/sql3/planner/oppqldistinctscan.go b/sql3/planner/oppqldistinctscan.go index 3ff84d0c6..e1f6d51e8 100644 --- a/sql3/planner/oppqldistinctscan.go +++ b/sql3/planner/oppqldistinctscan.go @@ -14,7 +14,6 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" - "github.com/pkg/errors" ) // PlanOpPQLDistinctScan plan operator handles a PQL distinct scan diff --git a/sql3/planner/oppqltablescan.go b/sql3/planner/oppqltablescan.go index a56261871..2b9eba57a 100644 --- a/sql3/planner/oppqltablescan.go +++ b/sql3/planner/oppqltablescan.go @@ -13,7 +13,6 @@ import ( "github.com/featurebasedb/featurebase/v3/sql3" "github.com/featurebasedb/featurebase/v3/sql3/parser" "github.com/featurebasedb/featurebase/v3/sql3/planner/types" - "github.com/pkg/errors" ) // PlanOpPQLTableScan plan operator handles a PQL table scan diff --git a/view.go b/view.go index 8be79bb17..c71d5cea6 100644 --- a/view.go +++ b/view.go @@ -16,7 +16,6 @@ import ( "github.com/featurebasedb/featurebase/v3/pql" "github.com/featurebasedb/featurebase/v3/roaring" - "github.com/featurebasedb/featurebase/v3/stats" "github.com/featurebasedb/featurebase/v3/testhook" "github.com/featurebasedb/featurebase/v3/vprint" "github.com/pkg/errors"