From 7f6ea0e6e5eef76a6bea308743cb4965b4c1ab3c Mon Sep 17 00:00:00 2001 From: pokeeffe-molecula <85502298+pokeeffe-molecula@users.noreply.github.com> Date: Wed, 11 Jan 2023 20:37:00 -0600 Subject: [PATCH] introduce performance counters and system table fanout, plus refactor metrics (#2363) * performance counters * first cut of perf counters and system table fanout and a wire protocol * significantly refactored prometheus support; removed statsd and exprvar * removed node_id * put dax subquery test back * Change Translator.TranslateFieldIDs method to take a dax.TableKeyer There are a bunch of other calls to the Translator interface methods with currently take an `index string`, and those need to be converted to dax.TableKeyer as well. But I need to review each call, because in at least one place I noticed one being called with `result.Index` instead of with the qtbl available. And I don't yet know how those could be different. Co-authored-by: Travis Turner --- api.go | 48 +- batch/batch.go | 8 +- batch/metrics.go | 25 - cache.go | 51 +- client/client.go | 16 - client/importer.go | 4 - dax/queryer/orchestrator.go | 103 ++- dax/queryer/queryer.go | 2 - dax/queryer/translator.go | 10 +- dax/test/dax/dax_test.go | 1 - executor.go | 104 ++- field.go | 4 - fragment.go | 19 +- go.mod | 2 +- holder.go | 15 - http_handler.go | 110 ++- idk/ingest.go | 71 +- idk/ingest_test.go | 2 +- idk/mds/importer.go | 2 - idk/metrics.go | 44 ++ importer.go | 2 - index.go | 7 +- metrics.go | 996 ++++++++++++++++++++++++++++ performancecounters.go | 203 ++++++ prometheus/prometheus.go | 307 --------- prometheus/prometheus_test.go | 50 +- server.go | 36 +- server/grpc.go | 43 +- server/handler_test.go | 9 - server/server.go | 28 - sql3/interfaces.go | 6 + sql3/planner/compilecreatetable.go | 8 +- sql3/planner/compileselect.go | 9 +- sql3/planner/executionplanner.go | 234 +++++++ sql3/planner/opbulkinsert.go | 13 + sql3/planner/opfanout.go | 114 ++++ sql3/planner/opinsert.go | 3 + sql3/planner/opsystemtable.go | 98 ++- sql3/planner/types/operator.go | 5 +- sql3/planner/wireprotocol.go | 211 ++++++ sql3/sql_complex_test.go | 140 ++-- stats/stats.go | 273 -------- stats/stats_test.go | 257 ------- statsd/statsd.go | 151 ----- statsd/statsd_test.go | 49 -- view.go | 4 - wireprotocol/wireprimitives.go | 563 ++++++++++++++++ wireprotocol/wireprimitives_test.go | 144 ++++ 48 files changed, 3074 insertions(+), 1530 deletions(-) create mode 100644 performancecounters.go delete mode 100644 prometheus/prometheus.go create mode 100644 sql3/planner/opfanout.go create mode 100644 sql3/planner/wireprotocol.go delete mode 100644 stats/stats.go delete mode 100644 stats/stats_test.go delete mode 100644 statsd/statsd.go delete mode 100644 statsd/statsd_test.go create mode 100644 wireprotocol/wireprimitives.go create mode 100644 wireprotocol/wireprimitives_test.go diff --git a/api.go b/api.go index 0e7ad39f7..e78bb967b 100644 --- a/api.go +++ b/api.go @@ -28,12 +28,12 @@ import ( "github.com/molecula/featurebase/v3/disco" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/rbf" + "github.com/prometheus/client_golang/prometheus" //"github.com/molecula/featurebase/v3/pg" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/roaring" planner_types "github.com/molecula/featurebase/v3/sql3/planner/types" - "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/tracing" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -277,7 +277,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index return nil, errors.Wrap(err, "creating index") } - api.holder.Stats.Count(MetricCreateIndex, 1, 1.0) + CounterCreateIndex.Inc() return index, nil } @@ -318,7 +318,7 @@ func (api *API) DeleteDataframe(ctx context.Context, indexName string) error { api.server.logger.Errorf("problem sending DeleteIndex message: %s", err) return errors.Wrap(err, "sending DeleteIndex message") } - api.holder.Stats.Count(MetricDeleteDataframe, 1, 1.0) + CounterDeleteDataframe.Inc() return nil } @@ -353,7 +353,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error { return errors.Wrap(err, "deleting id allocation for index") } } - api.holder.Stats.Count(MetricDeleteIndex, 1, 1.0) + CounterDeleteIndex.Inc() return nil } @@ -405,7 +405,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str return nil, errors.Wrap(err, "sending CreateField message") } - api.holder.Stats.CountWithCustomTags(MetricCreateField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + CounterCreateField.With(prometheus.Labels{"index": indexName}) return field, nil } @@ -749,7 +749,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str api.server.logger.Errorf("problem sending DeleteField message: %s", err) return errors.Wrap(err, "sending DeleteField message") } - api.holder.Stats.CountWithCustomTags(MetricDeleteField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + CounterDeleteField.With(prometheus.Labels{"index": indexName}) return nil } @@ -781,7 +781,7 @@ func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName str api.server.logger.Errorf("problem sending DeleteAvailableShard message: %s", err) return errors.Wrap(err, "sending DeleteAvailableShard message") } - api.holder.Stats.CountWithCustomTags(MetricDeleteAvailableShard, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)}) + CounterDeleteAvailableShard.With(prometheus.Labels{"index": indexName}).Inc() return nil } @@ -2095,15 +2095,6 @@ func (api *API) AvailableShards(ctx context.Context, indexName string) (*roaring return index.AvailableShards(false), nil } -// StatsWithTags returns an instance of whatever implementation of StatsClient -// pilosa is using with the given tags. -func (api *API) StatsWithTags(tags []string) stats.StatsClient { - if api.holder == nil || api.cluster == nil { - return nil - } - return api.holder.Stats.WithTags(tags...) -} - // LongQueryTime returns the configured threshold for logging/statting // long running queries. func (api *API) LongQueryTime() time.Duration { @@ -2390,19 +2381,19 @@ func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Du switch err { case nil: if exclusive { - api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0) + CounterExclusiveTransactionRequest.Inc() } else { - api.holder.Stats.Count(MetricTransactionStart, 1, 1.0) + CounterTransactionStart.Inc() } case ErrTransactionExclusive: if exclusive { - api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0) + CounterExclusiveTransactionBlocked.Inc() } else { - api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0) + CounterTransactionBlocked.Inc() } } if exclusive && t != nil && t.Active { - api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0) + CounterExclusiveTransactionActive.Inc() } return t, err } @@ -2414,9 +2405,9 @@ func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) ( t, err := api.server.FinishTransaction(ctx, id, remote) if err == nil { if t.Exclusive { - api.holder.Stats.Count(MetricExclusiveTransactionEnd, 1, 1.0) + CounterExclusiveTransactionEnd.Inc() } else { - api.holder.Stats.Count(MetricTransactionEnd, 1, 1.0) + CounterTransactionEnd.Inc() } } return t, err @@ -2436,7 +2427,7 @@ func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Tr t, err := api.server.GetTransaction(ctx, id, remote) if err == nil { if t.Exclusive && t.Active { - api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0) + CounterExclusiveTransactionActive.Inc() } } return t, err @@ -3045,6 +3036,10 @@ func (api *API) CompilePlan(ctx context.Context, q string) (planner_types.PlanOp return api.server.CompileExecutionPlan(ctx, q) } +func (api *API) RehydratePlanOperator(ctx context.Context, reader io.Reader) (planner_types.PlanOperator, error) { + return api.server.RehydratePlanOperator(ctx, reader) +} + func (api *API) RBFDebugInfo() map[string]*rbf.DebugInfo { infos := make(map[string]*rbf.DebugInfo) @@ -3342,6 +3337,7 @@ type SystemAPI interface { ClusterState() string DataDir() string + NodeID() string ClusterNodes() []ClusterNode } @@ -3413,6 +3409,10 @@ func (fsapi *FeatureBaseSystemAPI) DataDir() string { return fsapi.server.dataDir } +func (fsapi *FeatureBaseSystemAPI) NodeID() string { + return fsapi.cluster.Node.ID +} + func (fsapi *FeatureBaseSystemAPI) ClusterNodes() []ClusterNode { result := make([]ClusterNode, 0) diff --git a/batch/batch.go b/batch/batch.go index 80b06af24..8ee02d41c 100644 --- a/batch/batch.go +++ b/batch/batch.go @@ -755,7 +755,7 @@ func (b *Batch) Import() error { }() } defer func() { - b.importer.StatsTiming(MetricBatchImportDurationSeconds, time.Since(start), 1.0) + featurebase.SummaryBatchImportDurationSeconds.Observe(time.Since(start).Seconds()) }() size := len(b.ids) @@ -828,7 +828,7 @@ func (b *Batch) Flush() error { if err != nil { b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl) } - b.importer.StatsTiming(MetricBatchFlushDurationSeconds, time.Since(start), 1.0) + featurebase.SummaryBatchFlushDurationSeconds.Observe(time.Since(start).Seconds()) }() importStart := time.Now() @@ -1188,7 +1188,7 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error { } } - b.importer.StatsTiming(MetricBatchShardImportBuildRequestsSeconds, time.Since(start), 1.0) + featurebase.SummaryBatchShardImportBuildRequestsSeconds.Observe(time.Since(start).Seconds()) start = time.Now() eg := egpool.Group{PoolSize: 20} for shard, request := range requests { @@ -1200,7 +1200,7 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error { } err := eg.Wait() dur := time.Since(start) - b.importer.StatsTiming(MetricBatchShardImportDurationSeconds, dur, 1.0) + featurebase.SummaryBatchImportDurationSeconds.Observe(dur.Seconds()) b.log.Printf("import shard took: %v\n", dur) return errors.Wrap(err, "doing shard-transactional imports") } diff --git a/batch/metrics.go b/batch/metrics.go index f46277265..389c4ab2a 100644 --- a/batch/metrics.go +++ b/batch/metrics.go @@ -1,26 +1 @@ package batch - -const ( - // MetricBatchImportDurationSeconds records the full time of the - // RecordBatch.Import call. This includes starting and finishing a - // transaction, doing key translation, building fragments locally, - // importing all data, and resetting internal structures. - MetricBatchImportDurationSeconds = "batch_import_duration_seconds" - - // MetricBatchFlushDurationSeconds records the full time for - // RecordBatch.Flush (if splitBatchMode is in use). This includes - // starting and finishing a transaction, importing all data, and - // resetting internal structures. - MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds" - - // MetricBatchShardImportBuildRequestsSeconds is the time it takes - // after making fragments to build the shard-transactional request - // objects (but not actually import them or do any network activity). - MetricBatchShardImportBuildRequestsSeconds = "batch_shard_import_build_requests_seconds" - - // MetricBatchShardImportDurationSeconds is the time it takes to - // import all data for all shards in the batch using the - // shard-transactional endpoint. This does not include the time it - // takes to build the requests locally. - MetricBatchShardImportDurationSeconds = "batch_shard_import_duration_seconds" -) diff --git a/cache.go b/cache.go index c88b841eb..b285500bb 100644 --- a/cache.go +++ b/cache.go @@ -12,7 +12,6 @@ import ( "github.com/molecula/featurebase/v3/lru" pb "github.com/molecula/featurebase/v3/proto" - "github.com/molecula/featurebase/v3/stats" "github.com/pkg/errors" ) @@ -40,9 +39,6 @@ type cache interface { // Returns an ordered list of the top ranked bitmaps. Top() []bitmapPair - // SetStats defines the stats client used in the cache. - SetStats(s stats.StatsClient) - // Clear removes everything from the cache. If possible it should leave allocated structures in place to be reused. Clear() } @@ -51,7 +47,6 @@ type cache interface { type lruCache struct { cache *lru.Cache counts map[uint64]uint64 - stats stats.StatsClient // maxEntries is saved to support Clear which recreates the cache. maxEntries uint32 } @@ -61,7 +56,6 @@ func newLRUCache(maxEntries uint32) *lruCache { c := &lruCache{ cache: lru.New(int(maxEntries)), counts: make(map[uint64]uint64), - stats: stats.NopStatsClient, maxEntries: maxEntries, } c.cache.OnEvicted = c.onEvicted @@ -119,11 +113,6 @@ func (c *lruCache) Top() []bitmapPair { return a } -// SetStats defines the stats client used in the cache. -func (c *lruCache) SetStats(s stats.StatsClient) { - c.stats = s -} - func (c *lruCache) Clear() { for k := range c.counts { delete(c.counts, k) @@ -157,8 +146,6 @@ type rankCache struct { // thresholdValue is the value of the last item in the cache thresholdValue uint64 - - stats stats.StatsClient } // NewRankCache returns a new instance of RankCache. @@ -167,7 +154,6 @@ func NewRankCache(maxEntries uint32) *rankCache { maxEntries: maxEntries, thresholdBuffer: int(thresholdFactor * float64(maxEntries)), entries: make(map[uint64]uint64), - stats: stats.NopStatsClient, } } @@ -228,7 +214,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) { // as this can take up an upbounded amount of memory. This is especially // true when restoring shards as all rows will be added. if len(c.entries) > int(2*c.maxEntries) { - c.stats.Count(MetricRecalculateCache, 1, 1.0) + CounterRecalculateCache.Inc() c.recalculate() } } @@ -273,7 +259,7 @@ func (c *rankCache) Invalidate() { func (c *rankCache) Recalculate() { c.mu.Lock() defer c.mu.Unlock() - c.stats.Count(MetricRecalculateCache, 1, 1.0) + CounterRecalculateCache.Inc() c.recalculate() } @@ -285,12 +271,12 @@ func (c *rankCache) invalidate() { // This is somewhat necessary for now since recalculation is not cheap. // The cache will remain flagged as dirty and will be recalculated if Top is called. // This may cause unexpected memory growth, so record it in metrics for debugging purposes. - c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0) + CounterInvalidateCacheSkipped.Inc() // Ensure that we're marked as dirty even if we weren't otherwise. c.dirty = true return } - c.stats.Count(MetricInvalidateCache, 1, 1.0) + CounterInvalidateCache.Inc() c.recalculate() } @@ -316,7 +302,7 @@ func (c *rankCache) recalculate() { // Store the count of the item at the threshold index. length := len(c.rankings) - c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0) + GaugeRankCacheLength.Set(float64(length)) var removeItems []bitmapPair // cached, ordered list if length > int(c.maxEntries) { @@ -332,7 +318,7 @@ func (c *rankCache) recalculate() { // If size is larger than the threshold then trim it. if len(c.entries) > c.thresholdBuffer { - c.stats.Count(MetricCacheThresholdReached, 1, 1.0) + CounterCacheThresholdReached.Inc() for _, pair := range removeItems { delete(c.entries, pair.ID) } @@ -342,11 +328,6 @@ func (c *rankCache) recalculate() { c.dirty = false } -// SetStats defines the stats client used in the cache. -func (c *rankCache) SetStats(s stats.StatsClient) { - c.stats = s -} - // Top returns an ordered list of pairs. func (c *rankCache) Top() []bitmapPair { c.mu.Lock() @@ -354,7 +335,7 @@ func (c *rankCache) Top() []bitmapPair { if c.dirty { // The cache is dirty, so we need to recalculate it to get a consistent view. - c.stats.Count(MetricReadDirtyCache, 1, 1.0) + CounterReadDirtyCache.Inc() c.recalculate() } @@ -605,25 +586,21 @@ func (p uint64Slice) Len() int { return len(p) } func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } // nopCache represents a no-op Cache implementation. -type nopCache struct { - stats stats.StatsClient -} +type nopCache struct{} // Ensure NopCache implements Cache. -var globalNopCache cache = nopCache{ - stats: stats.NopStatsClient, -} +var globalNopCache cache = nopCache{} func (c nopCache) Add(uint64, uint64) {} func (c nopCache) BulkAdd(uint64, uint64) {} func (c nopCache) Get(uint64) uint64 { return 0 } func (c nopCache) IDs() []uint64 { return []uint64{} } -func (c nopCache) Invalidate() {} -func (c nopCache) Len() int { return 0 } -func (c nopCache) Recalculate() {} -func (c nopCache) SetStats(stats.StatsClient) {} -func (c nopCache) Clear() {} +func (c nopCache) Invalidate() {} +func (c nopCache) Len() int { return 0 } +func (c nopCache) Recalculate() {} + +func (c nopCache) Clear() {} func (c nopCache) Top() []bitmapPair { return []bitmapPair{} diff --git a/client/client.go b/client/client.go index 2a831442a..004990fc3 100644 --- a/client/client.go +++ b/client/client.go @@ -29,7 +29,6 @@ import ( "github.com/molecula/featurebase/v3/pb" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/roaring" - "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/vprint" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -52,7 +51,6 @@ type Client struct { manualFragmentNode *fragmentNode manualServerURI *pnet.URI tracer opentracing.Tracer - Stats stats.StatsClient // An exponential backoff algorithm retries requests exponentially (if an HTTP request fails), // increasing the waiting time between retries up to a maximum backoff time. maxBackoff time.Duration @@ -211,11 +209,6 @@ func newClientWithOptions(options *ClientOptions) *Client { } else { c.tracer = options.tracer } - if options.stats == nil { - c.Stats = stats.NopStatsClient - } else { - c.Stats = options.stats - } c.maxRetries = *options.retries c.maxBackoff = 2 * time.Minute @@ -1359,7 +1352,6 @@ type ClientOptions struct { manualServerAddress bool tracer opentracing.Tracer retries *int - stats stats.StatsClient nat map[pnet.URI]pnet.URI pathPrefix string } @@ -1445,14 +1437,6 @@ func OptClientRetries(retries int) ClientOption { } } -// OptClientStatsClient sets a stats client, such as Prometheus -func OptClientStatsClient(stats stats.StatsClient) ClientOption { - return func(options *ClientOptions) error { - options.stats = stats - return nil - } -} - // OptClientNAT sets a NAT map used to translate the advertised URI to something // else (for example, when accessing pilosa running in docker). func OptClientNAT(nat map[string]string) ClientOption { diff --git a/client/importer.go b/client/importer.go index c42c74c4d..09459c466 100644 --- a/client/importer.go +++ b/client/importer.go @@ -328,7 +328,3 @@ func (i *importer) EncodeImport(ctx context.Context, tid dax.TableID, fld *dax.F func (i *importer) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, path string, data []byte) error { return i.client.DoImport(string(tid), shard, path, data) } - -func (i *importer) StatsTiming(name string, value time.Duration, rate float64) { - i.client.Stats.Timing(name, value, rate) -} diff --git a/dax/queryer/orchestrator.go b/dax/queryer/orchestrator.go index d5a74370b..4396f9279 100644 --- a/dax/queryer/orchestrator.go +++ b/dax/queryer/orchestrator.go @@ -14,8 +14,8 @@ import ( "github.com/molecula/featurebase/v3/errors" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/pql" - "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/tracing" + "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" ) @@ -74,7 +74,7 @@ type Translator interface { // TODO(jaffee) the naming here is a cluster. TranslateIndexIDs takes a list, but TranslateFieldIDs takes a set, both have alternate methods that take the other thing. :facepalm: TranslateIndexIDs(ctx context.Context, index string, ids []uint64) ([]string, error) TranslateIndexIDSet(ctx context.Context, index string, ids map[uint64]struct{}) (map[uint64]string, error) - TranslateFieldIDs(ctx context.Context, index, field string, ids map[uint64]struct{}) (map[uint64]string, error) + TranslateFieldIDs(ctx context.Context, tableKeyer dax.TableKeyer, field string, ids map[uint64]struct{}) (map[uint64]string, error) TranslateFieldListIDs(ctx context.Context, index, field string, ids []uint64) ([]string, error) } @@ -87,7 +87,6 @@ type orchestrator struct { // Client used for remote requests. client *featurebase.InternalClient - stats stats.StatsClient logger logger.Logger } @@ -433,11 +432,11 @@ func (o *orchestrator) executeCall(ctx context.Context, tableKeyer dax.TableKeye } else if err := o.validateCallArgs(c); err != nil { return nil, errors.Wrap(err, "validating args") } - indexTag := "index:" + string(tableKeyer.Key()) - metricName := "query_" + strings.ToLower(c.Name) + "_total" - statFn := func() { + + labels := prometheus.Labels{"index": string(tableKeyer.Key())} + statFn := func(ctr *prometheus.CounterVec) { if !opt.Remote { - o.stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag}) + ctr.With(labels).Inc() } } @@ -449,101 +448,106 @@ func (o *orchestrator) executeCall(ctx context.Context, tableKeyer dax.TableKeye switch c.Name { case "Sum": - statFn() + statFn(featurebase.CounterQuerySumTotal) res, err := o.executeSum(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeSum") case "Min": - statFn() + statFn(featurebase.CounterQueryMinTotal) res, err := o.executeMin(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeMin") case "Max": - statFn() + statFn(featurebase.CounterQueryMaxTotal) res, err := o.executeMax(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeMax") case "MinRow": - statFn() + statFn(featurebase.CounterQueryMinRowTotal) res, err := o.executeMinRow(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeMinRow") case "MaxRow": - statFn() + statFn(featurebase.CounterQueryMaxRowTotal) res, err := o.executeMaxRow(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeMaxRow") // case "Clear": - // statFn() + // statFn(featurebase.CounterQueryClearTotal) // res, err := o.executeClearBit(ctx, index, c, opt) // return res, errors.Wrap(err, "executeClearBit") // case "ClearRow": - // statFn() + // statFn(featurebase.CounterQueryClearRowTotal) // res, err := o.executeClearRow(ctx, index, c, shards, opt) // return res, errors.Wrap(err, "executeClearRow") case "Distinct": - statFn() + statFn(featurebase.CounterQueryDistinctTotal) res, err := o.executeDistinct(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeDistinct") // case "Store": - // statFn() + // statFn(featurebase.CounterQueryStoreTotal) // res, err := o.executeSetRow(ctx, index, c, shards, opt) // return res, errors.Wrap(err, "executeSetRow") case "Count": - statFn() + statFn(featurebase.CounterQueryCountTotal) res, err := o.executeCount(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeCount") // case "Set": - // statFn() + // statFn(featurebase.CounterQuerySetTotal) // res, err := o.executeSet(ctx, index, c, opt) // return res, errors.Wrap(err, "executeSet") case "TopK": - statFn() + statFn(featurebase.CounterQueryTopKTotal) res, err := o.executeTopK(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeTopK") case "TopN": - statFn() + statFn(featurebase.CounterQueryTopNTotal) res, err := o.executeTopN(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeTopN") case "Rows": - statFn() + statFn(featurebase.CounterQueryRowsTotal) res, err := o.executeRows(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeRows") case "Extract": - statFn() + statFn(featurebase.CounterQueryExtractTotal) res, err := o.executeExtract(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeExtract") case "GroupBy": - statFn() + statFn(featurebase.CounterQueryGroupByTotal) res, err := o.executeGroupBy(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeGroupBy") case "Options": - statFn() + statFn(featurebase.CounterQueryOptionsTotal) res, err := o.executeOptionsCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeOptionsCall") case "IncludesColumn": + statFn(featurebase.CounterQueryIncludesColumnTotal) res, err := o.executeIncludesColumnCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeIncludesColumnCall") case "FieldValue": - statFn() + statFn(featurebase.CounterQueryFieldValueTotal) res, err := o.executeFieldValueCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeFieldValueCall") case "Precomputed": + statFn(featurebase.CounterQueryPrecomputedTotal) res, err := o.executePrecomputedCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executePrecomputedCall") case "UnionRows": + statFn(featurebase.CounterQueryUnionRowsTotal) res, err := o.executeUnionRows(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeUnionRows") case "ConstRow": + statFn(featurebase.CounterQueryConstRowTotal) res, err := o.executeConstRow(ctx, tableKeyer, c) return res, errors.Wrap(err, "executeConstRow") case "Limit": + statFn(featurebase.CounterQueryLimitTotal) res, err := o.executeLimitCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeLimitCall") case "Percentile": + statFn(featurebase.CounterQueryPercentileTotal) res, err := o.executePercentile(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executePercentile") // case "Delete": - // statFn() //TODO(twg) need this? + // statFn(featurebase.CounterQueryDeleteTotal) // res, err := o.executeDeleteRecords(ctx, index, c, shards, opt) // return res, errors.Wrap(err, "executeDelete") default: // o.g. "Row", "Union", "Intersect" or anything that returns a bitmap. - statFn() res, err := o.executeBitmapCall(ctx, tableKeyer, c, shards, opt) return res, errors.Wrap(err, "executeBitmapCall") } @@ -1116,13 +1120,42 @@ func (o *orchestrator) executeBitmapCall(ctx context.Context, tableKeyer dax.Tab span.LogKV("pqlCallName", c.Name) defer span.Finish() - indexTag := "index:" + string(tableKeyer.Key()) - metricName := "query_" + strings.ToLower(c.Name) + "_total" - if c.Name == "Row" && c.HasConditionArg() { - metricName = "query_row_bsi_total" + labels := prometheus.Labels{"index": string(tableKeyer.Key())} + statFn := func(ctr *prometheus.CounterVec) { + if !opt.Remote { + ctr.With(labels).Inc() + } } + if !opt.Remote { - o.stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag}) + switch c.Name { + case "Row": + if c.HasConditionArg() { + statFn(featurebase.CounterQueryRowBSITotal) + } else { + statFn(featurebase.CounterQueryRowTotal) + } + case "Range": + statFn(featurebase.CounterQueryRangeTotal) + case "Difference": + statFn(featurebase.CounterQueryBitmapTotal) + case "Intersect": + statFn(featurebase.CounterQueryIntersectTotal) + case "Union": + statFn(featurebase.CounterQueryUnionTotal) + case "InnerUnionRows": + statFn(featurebase.CounterQueryInnerUnionRowsTotal) + case "Xor": + statFn(featurebase.CounterQueryXorTotal) + case "Not": + statFn(featurebase.CounterQueryNotTotal) + case "Shift": + statFn(featurebase.CounterQueryShiftTotal) + case "All": + statFn(featurebase.CounterQueryAllTotal) + default: + statFn(featurebase.CounterQueryBitmapTotal) + } } // Merge returned results at coordinating node. @@ -2985,9 +3018,7 @@ func (o *orchestrator) preTranslateMatrixSet(ctx context.Context, mat featurebas } } - index := string(tableKeyer.Key()) - - return o.trans.TranslateFieldIDs(ctx, index, field, ids) + return o.trans.TranslateFieldIDs(ctx, tableKeyer, field, ids) } func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedTable, call *pql.Call, result interface{}, idSet map[uint64]string) (_ interface{}, err error) { @@ -3164,7 +3195,7 @@ func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedT fieldTranslations := make(map[string]map[uint64]string) for field, ids := range fieldIDs { - trans, err := o.trans.TranslateFieldIDs(ctx, idx.Name, field.Name, ids) + trans, err := o.trans.TranslateFieldIDs(ctx, qtbl, field.Name, ids) if err != nil { return nil, errors.Wrapf(err, "translating IDs in field '%q'", field.Name) } diff --git a/dax/queryer/queryer.go b/dax/queryer/queryer.go index ccb7f9834..8d6040193 100644 --- a/dax/queryer/queryer.go +++ b/dax/queryer/queryer.go @@ -22,7 +22,6 @@ import ( "github.com/molecula/featurebase/v3/sql3/parser" "github.com/molecula/featurebase/v3/sql3/planner" plannertypes "github.com/molecula/featurebase/v3/sql3/planner/types" - "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/systemlayer" uuid "github.com/satori/go.uuid" ) @@ -89,7 +88,6 @@ func (q *Queryer) Orchestrator(qual dax.TableQualifier) *qualifiedOrchestrator { topology: &MDSTopology{noder: q.noder}, // TODO(jaffee) using default http.Client probably bad... need to set some timeouts. client: q.fbClient, - stats: stats.NopStatsClient, logger: q.logger, } diff --git a/dax/queryer/translator.go b/dax/queryer/translator.go index 191d3bc8d..3da33c9b3 100644 --- a/dax/queryer/translator.go +++ b/dax/queryer/translator.go @@ -229,15 +229,19 @@ func (m *mdsTranslator) TranslateIndexIDSet(ctx context.Context, table string, i } return ret, nil } -func (m *mdsTranslator) TranslateFieldIDs(ctx context.Context, table, field string, ids map[uint64]struct{}) (map[uint64]string, error) { +func (m *mdsTranslator) TranslateFieldIDs(ctx context.Context, tableKeyer dax.TableKeyer, field string, ids map[uint64]struct{}) (map[uint64]string, error) { idList := make([]uint64, 0, len(ids)) for id := range ids { idList = append(idList, id) } - stringList, err := m.TranslateFieldListIDs(ctx, table, field, idList) + // TODO(tlt): convert TranslateFieldListIDs (and the other Translator + // interface methods) to TableKeyer. + index := string(tableKeyer.Key()) + + stringList, err := m.TranslateFieldListIDs(ctx, index, field, idList) if err != nil { - return nil, errors.Wrapf(err, "translating field ids on field: %s, %s", table, field) + return nil, errors.Wrapf(err, "translating field ids on field: %s, %s", tableKeyer, field) } ret := make(map[uint64]string) diff --git a/dax/test/dax/dax_test.go b/dax/test/dax/dax_test.go index 9541f34ad..978c50c36 100644 --- a/dax/test/dax/dax_test.go +++ b/dax/test/dax/dax_test.go @@ -112,7 +112,6 @@ func TestDAXIntegration(t *testing.T) { "alterTable/alterTableBadTable", // looks like table does not exist is a different error in DAX "top-tests/test-1", // don't know why this is failing at all "delete_tests", - "subquerytable", // subqueries seem to be a problem } doSkip := func(name string) bool { diff --git a/executor.go b/executor.go index 4488ce628..91aa6d32d 100644 --- a/executor.go +++ b/executor.go @@ -28,6 +28,7 @@ import ( "github.com/molecula/featurebase/v3/testhook" "github.com/molecula/featurebase/v3/tracing" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" ) @@ -164,16 +165,16 @@ func (e *executor) Close() error { // PoolSize is exported to let the task pool update us func (e *executor) PoolSize(n int) { if e.Holder != nil { - e.Holder.Stats.Gauge("worker_total", float64(n), 0) + GaugeWorkerTotal.Set(float64(n)) } } // InitStats initializes stats counters. Must be called after Holder set. func (e *executor) InitStats() { if e.Holder != nil { - e.Holder.Stats.Count("job_total", 0, 0) + CounterJobTotal.Add(0) l, _, _ := e.workers.Stats() - e.Holder.Stats.Gauge("worker_total", float64(l), 0) + GaugeWorkerTotal.Set(float64(l)) } } @@ -683,11 +684,11 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p } else if err := e.validateCallArgs(c); err != nil { return nil, errors.Wrap(err, "validating args") } - indexTag := "index:" + index - metricName := "query_" + strings.ToLower(c.Name) + "_total" - statFn := func() { + + labels := prometheus.Labels{"index": index} + statFn := func(ctr *prometheus.CounterVec) { if !opt.Remote { - e.Holder.Stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag}) + ctr.With(labels).Inc() } } @@ -719,114 +720,122 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p switch c.Name { case "Sum": - statFn() + statFn(CounterQuerySumTotal) res, err := e.executeSum(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeSum") case "Min": - statFn() + statFn(CounterQueryMinTotal) res, err := e.executeMin(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeMin") case "Max": - statFn() + statFn(CounterQueryMaxTotal) res, err := e.executeMax(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeMax") case "MinRow": - statFn() + statFn(CounterQueryMinRowTotal) res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeMinRow") case "MaxRow": - statFn() + statFn(CounterQueryMaxRowTotal) res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeMaxRow") case "Clear": - statFn() + statFn(CounterQueryClearTotal) res, err := e.executeClearBit(ctx, qcx, index, c, opt) return res, errors.Wrap(err, "executeClearBit") case "ClearRow": - statFn() + statFn(CounterQueryClearRowTotal) res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeClearRow") case "Distinct": - statFn() + statFn(CounterQueryDistinctTotal) res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeDistinct") case "Store": - statFn() + statFn(CounterQueryStoreTotal) res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeSetRow") case "Count": - statFn() + statFn(CounterQueryCountTotal) res, err := e.executeCount(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeCount") case "Set": - statFn() + statFn(CounterQuerySetTotal) res, err := e.executeSet(ctx, qcx, index, c, opt) return res, errors.Wrap(err, "executeSet") case "TopK": - statFn() + statFn(CounterQueryTopKTotal) res, err := e.executeTopK(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeTopK") case "TopN": - statFn() + statFn(CounterQueryTopNTotal) res, err := e.executeTopN(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeTopN") case "Rows": - statFn() + statFn(CounterQueryRowsTotal) res, err := e.executeRows(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeRows") case "ExternalLookup": - statFn() + statFn(CounterQueryExternalLookupTotal) res, err := e.executeExternalLookup(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeExternalLookup") case "Extract": - statFn() + statFn(CounterQueryExtractTotal) res, err := e.executeExtract(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeExtract") case "GroupBy": - statFn() + statFn(CounterQueryGroupByTotal) res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeGroupBy") case "Options": - statFn() + statFn(CounterQueryOptionsTotal) res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeOptionsCall") case "IncludesColumn": + statFn(CounterQueryIncludesColumnTotal) res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeIncludesColumnCall") case "FieldValue": - statFn() + statFn(CounterQueryFieldValueTotal) res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeFieldValueCall") case "Precomputed": + statFn(CounterQueryPrecomputedTotal) res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executePrecomputedCall") case "UnionRows": + statFn(CounterQueryUnionRowsTotal) res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeUnionRows") case "ConstRow": + statFn(CounterQueryConstRowTotal) res, err := e.executeConstRow(ctx, index, c) return res, errors.Wrap(err, "executeConstRow") case "Limit": + statFn(CounterQueryLimitTotal) res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeLimitCall") case "Percentile": + statFn(CounterQueryPercentileTotal) res, err := e.executePercentile(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executePercentile") case "Delete": - statFn() // TODO(twg) need this? + statFn(CounterQueryDeleteTotal) res, err := e.executeDeleteRecords(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeDelete") case "Sort": + statFn(CounterQuerySortTotal) res, err := e.executeSort(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeSort") case "Apply": + statFn(CounterQueryApplyTotal) res, err := e.executeApply(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeApply") case "Arrow": + statFn(CounterQueryArrowTotal) res, err := e.executeArrow(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeArrow") default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap. - statFn() res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt) return res, errors.Wrap(err, "executeBitmapCall") } @@ -1505,13 +1514,42 @@ func (e *executor) executeBitmapCall(ctx context.Context, qcx *Qcx, index string span.LogKV("pqlCallName", c.Name) defer span.Finish() - indexTag := "index:" + index - metricName := "query_" + strings.ToLower(c.Name) + "_total" - if c.Name == "Row" && c.HasConditionArg() { - metricName = "query_row_bsi_total" + labels := prometheus.Labels{"index": index} + statFn := func(ctr *prometheus.CounterVec) { + if !opt.Remote { + ctr.With(labels).Inc() + } } + if !opt.Remote { - e.Holder.Stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag}) + switch c.Name { + case "Row": + if c.HasConditionArg() { + statFn(CounterQueryRowBSITotal) + } else { + statFn(CounterQueryRowTotal) + } + case "Range": + statFn(CounterQueryRangeTotal) + case "Difference": + statFn(CounterQueryBitmapTotal) + case "Intersect": + statFn(CounterQueryIntersectTotal) + case "Union": + statFn(CounterQueryUnionTotal) + case "InnerUnionRows": + statFn(CounterQueryInnerUnionRowsTotal) + case "Xor": + statFn(CounterQueryXorTotal) + case "Not": + statFn(CounterQueryNotTotal) + case "Shift": + statFn(CounterQueryShiftTotal) + case "All": + statFn(CounterQueryAllTotal) + default: + statFn(CounterQueryBitmapTotal) + } } // Execute calls in bulk on each remote node and merge. diff --git a/field.go b/field.go index 55d6b3543..ea4772899 100644 --- a/field.go +++ b/field.go @@ -17,7 +17,6 @@ import ( "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/roaring" - "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/testhook" "github.com/molecula/featurebase/v3/tracing" "github.com/pkg/errors" @@ -83,7 +82,6 @@ type Field struct { viewMap map[string]*view broadcaster broadcaster - Stats stats.StatsClient serializer Serializer // Field options. @@ -395,7 +393,6 @@ func newField(holder *Holder, path, index, name string, opts FieldOption) (*Fiel viewMap: make(map[string]*view), broadcaster: NopBroadcaster, - Stats: stats.NopStatsClient, serializer: NopSerializer, options: applyDefaultOptions(&fo), @@ -1164,7 +1161,6 @@ func (f *Field) newView(path, name string) *view { view := newView(f.holder, path, f.index, f.name, name, f.options) view.idx = f.idx view.fld = f - view.stats = f.Stats view.broadcaster = f.broadcaster return view } diff --git a/fragment.go b/fragment.go index 7d2183009..47a146ded 100644 --- a/fragment.go +++ b/fragment.go @@ -24,7 +24,6 @@ import ( "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/roaring" "github.com/molecula/featurebase/v3/shardwidth" - "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/testhook" "github.com/molecula/featurebase/v3/tracing" "github.com/molecula/featurebase/v3/vprint" @@ -119,8 +118,6 @@ type fragment struct { // mutexVector is used for mutex field types. It's checked for an // existing value (to clear) prior to setting a new value. mutexVector vector - - stats stats.StatsClient } // newFragment returns a new instance of fragment. @@ -141,8 +138,6 @@ func newFragment(holder *Holder, idx *Index, fld *Field, vw *view, shard uint64) CacheSize: DefaultCacheSize, holder: holder, - - stats: stats.NopStatsClient, } return f } @@ -404,7 +399,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo f.cache.Add(rowID, n) } - f.stats.Count(MetricSetBit, 1, 1.0) + CounterSetBit.Inc() return changed, nil } @@ -453,7 +448,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b f.cache.Add(rowID, n) } - f.stats.Count(MetricClearBit, 1, 1.0) + CounterClearBit.Inc() return changed, nil } @@ -509,7 +504,7 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo } } - f.stats.Count("setRow", 1, 1.0) + CounterSetRow.Inc() return changed, nil } @@ -1712,23 +1707,23 @@ func (p parallelSlices) Swap(i, j int) { // operations to the op log. func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64]struct{}) error { if len(set) > 0 { - f.stats.Count(MetricImportingN, int64(len(set)), 1) + CounterImportingN.Add(float64(len(set))) // TODO benchmark Add/RemoveN behavior with sorted/unsorted positions changedN, err := tx.Add(f.index(), f.field(), f.view(), f.shard, set...) if err != nil { return errors.Wrap(err, "adding positions") } - f.stats.Count(MetricImportedN, int64(changedN), 1) + CounterImportedN.Add(float64(changedN)) } if len(clear) > 0 { - f.stats.Count(MetricClearingN, int64(len(clear)), 1) + CounterClearingingN.Add(float64(len(clear))) changedN, err := tx.Remove(f.index(), f.field(), f.view(), f.shard, clear...) if err != nil { return errors.Wrap(err, "clearing positions") } - f.stats.Count(MetricClearedN, int64(changedN), 1) + CounterClearedN.Add(float64(changedN)) } return f.updateCaching(tx, rowSet) } diff --git a/go.mod b/go.mod index 9dbe467e8..4548cccb7 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ replace robpike.io/ivy => github.com/tgruben/ivy v0.0.0-20221107170120-634b546dc require ( github.com/CAFxX/gcnotifier v0.0.0-20220409005548-0153238b886a - github.com/DataDog/datadog-go v4.8.3+incompatible + github.com/DataDog/datadog-go v4.8.3+incompatible // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2 diff --git a/holder.go b/holder.go index feeef5dcf..018c7a8b6 100644 --- a/holder.go +++ b/holder.go @@ -17,7 +17,6 @@ import ( "github.com/molecula/featurebase/v3/logger" rbfcfg "github.com/molecula/featurebase/v3/rbf/cfg" "github.com/molecula/featurebase/v3/roaring" - "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/storage" "github.com/molecula/featurebase/v3/testhook" "github.com/molecula/featurebase/v3/vprint" @@ -79,9 +78,6 @@ type Holder struct { wg sync.WaitGroup closing chan struct{} - // Stats - Stats stats.StatsClient - // Data directory path. path string @@ -256,7 +252,6 @@ type HolderConfig struct { Schemator disco.Schemator Sharder disco.Sharder CacheFlushInterval time.Duration - StatsClient stats.StatsClient Logger logger.Logger StorageConfig *storage.Config @@ -281,7 +276,6 @@ func DefaultHolderConfig() *HolderConfig { Schemator: disco.NewInMemSchemator(), Sharder: disco.InMemSharder, CacheFlushInterval: defaultCacheFlushInterval, - StatsClient: stats.NopStatsClient, Logger: logger.NopLogger, StorageConfig: storage.NewDefaultConfig(), RBFConfig: rbfcfg.NewDefaultConfig(), @@ -323,7 +317,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { broadcaster: NopBroadcaster, partitionN: cfg.PartitionN, - Stats: cfg.StatsClient, cacheFlushInterval: cfg.CacheFlushInterval, OpenTranslateStore: cfg.OpenTranslateStore, OpenTranslateReader: cfg.OpenTranslateReader, @@ -526,8 +519,6 @@ func (h *Holder) Open() error { // Check if deletion was in progress when server was shutdown h.processDeleteInflight() - h.Stats.Open() - h.opened.Close() _ = testhook.Opened(h.Auditor, h, nil) @@ -628,8 +619,6 @@ func (h *Holder) Close() error { fmt.Printf("%v\n", globalCallStats.report()) } - h.Stats.Close() - // Notify goroutines of closing and wait for completion. close(h.closing) h.wg.Wait() @@ -1156,7 +1145,6 @@ func (h *Holder) newIndex(path, name string) (*Index, error) { if err != nil { return nil, err } - index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name())) index.broadcaster = h.broadcaster index.serializer = h.serializer index.OpenTranslateStore = h.OpenTranslateStore @@ -1329,9 +1317,6 @@ type holderSyncer struct { syncers errgroup.Group - // Stats - Stats stats.StatsClient - // Signals that the sync should stop. Closing <-chan struct{} } diff --git a/http_handler.go b/http_handler.go index c17c2775d..cf779bd39 100644 --- a/http_handler.go +++ b/http_handler.go @@ -8,7 +8,6 @@ import ( "encoding/gob" "encoding/hex" "encoding/json" - "expvar" "fmt" "io" "math" @@ -42,8 +41,11 @@ import ( "github.com/molecula/featurebase/v3/rbf" "github.com/molecula/featurebase/v3/sql3/planner/types" "github.com/molecula/featurebase/v3/storage" + "github.com/molecula/featurebase/v3/wireprotocol" + "github.com/molecula/featurebase/v3/tracing" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" dto "github.com/prometheus/client_model/go" "github.com/prometheus/prom2json" @@ -401,8 +403,7 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { next.ServeHTTP(w, r) dur := time.Since(t) - statsTags := make([]string, 0, 5) - + isSlow := "false" longQueryTime := h.api.LongQueryTime() if longQueryTime > 0 && dur > longQueryTime { queryRequest := r.Context().Value(contextKeyQueryRequest) @@ -413,31 +414,27 @@ func (h *Handler) collectStats(next http.Handler) http.Handler { } h.logger.Printf("HTTP query duration %v exceeds %v: %s %s %s", dur, longQueryTime, r.Method, r.URL.String(), queryString) - statsTags = append(statsTags, "slow:true") - } else { - statsTags = append(statsTags, "slow:false") + isSlow = "true" } + where := "" pathParts := strings.Split(r.URL.Path, "/") if externalPrefixFlag[pathParts[1]] { - statsTags = append(statsTags, "where:external") + where = "external" } else { - statsTags = append(statsTags, "where:internal") + where = "internal" } - - statsTags = append(statsTags, "useragent:"+r.UserAgent()) - path, err := mux.CurrentRoute(r).GetPathTemplate() - if err == nil { - statsTags = append(statsTags, "path:"+path) - } - - statsTags = append(statsTags, "method:"+r.Method) - - stats := h.api.StatsWithTags(statsTags) - if stats != nil { - stats.Timing(MetricHTTPRequest, dur, 0.1) + if err != nil { + path = "" } + SummaryHttpRequests.With(prometheus.Labels{ + "method": r.Method, + "path": path, + "slow": isSlow, + "useragent": r.UserAgent(), + "where": where, + }).Observe(dur.Seconds()) }) } @@ -505,7 +502,6 @@ func newRouter(handler *Handler) http.Handler { // TODO: figure out how to protect these if needed router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.PathPrefix("/debug/fgprof").Handler(fgprof.Handler()).Methods("GET") - router.Handle("/debug/vars", expvar.Handler()).Methods("GET") router.Handle("/metrics", promhttp.Handler()) router.HandleFunc("/metrics.json", handler.chkAuthZ(handler.handleGetMetricsJSON, authz.Admin)).Methods("GET").Name("GetMetricsJSON") @@ -551,6 +547,8 @@ func newRouter(handler *Handler) http.Handler { if handler.sqlEnabled { router.HandleFunc("/sql", handler.chkAuthZ(handler.handlePostSQL, authz.Admin)).Methods("POST").Name("PostSQL") } + // internal endpoint + router.HandleFunc("/sql", handler.chkAuthZ(handler.handlePostSQLPlanOperator, authz.Admin)).Headers("X-FeatureBase-Plan-Operator", "").Methods("POST").Name("PostSQLPlanOperator") router.HandleFunc("/query-history", handler.chkAuthZ(handler.handleGetPastQueries, authz.Admin)).Methods("GET").Name("GetPastQueries") router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion") @@ -1392,10 +1390,70 @@ func (h *Handler) writeBadRequest(w http.ResponseWriter, r *http.Request, err er } } -// handlePostSQL handles /sql requests. -func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { - includePlan := false +// handlePostSQLOperator handles an internal sql3 plan operator execution request +// these requests come from other nodes in the cluster +// handlePostSQLOperator will 'rehydrate' a plan operator and return data in the +// featurebase wire format for effciency +// we do not track these requests as user requests +// TODO(pok) - thus is there anything we need here to align with how we do this for other nodes +func (h *Handler) handlePostSQLPlanOperator(w http.ResponseWriter, r *http.Request) { + writeError := func(err error) { + if err != nil { + w.Write(wireprotocol.WriteError(err)) + } + } + + // always finish with a done message + defer w.Write(wireprotocol.WriteDone()) + + ctx := r.Context() + + rootOperator, err := h.api.RehydratePlanOperator(ctx, r.Body) + if err != nil { + writeError(err) + return + } + + // get a query iterator. + iter, err := rootOperator.Iterator(ctx, nil) + if err != nil { + writeError(err) + return + } + // read schema & write to response. + columns := rootOperator.Schema() + b, err := wireprotocol.WriteSchema(columns) + if err != nil { + writeError(err) + return + } + w.Write(b) + + var rowErr error + var currentRow types.Row + var nextErr error + + for currentRow, nextErr = iter.Next(ctx); nextErr == nil; currentRow, nextErr = iter.Next(ctx) { + b, err := wireprotocol.WriteRow(currentRow, columns) + if err != nil { + rowErr = err + break + } + w.Write(b) + } + if nextErr != nil && nextErr != types.ErrNoMoreRows { + rowErr = nextErr + } + writeError(rowErr) +} + +// handlePostSQL handles /sql requests +// supports a ?plan=true|false parameter to send back the plan in the +// query response +func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { + + includePlan := false includePlanValue := r.URL.Query().Get("plan") if len(includePlanValue) > 0 { var err error @@ -1406,6 +1464,7 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { } } + // get the body b, err := io.ReadAll(r.Body) if err != nil { h.writeBadRequest(w, r, err) @@ -1419,6 +1478,9 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) { // put the requestId in the context ctx := fbcontext.WithRequestID(r.Context(), requestID.String()) + // update the counter for requests + PerfCounterSQLRequestSec.Add(1) + // Write response back to client. w.Header().Set("Content-Type", "application/json") diff --git a/idk/ingest.go b/idk/ingest.go index fe418f947..710e7654e 100644 --- a/idk/ingest.go +++ b/idk/ingest.go @@ -34,9 +34,7 @@ import ( "github.com/molecula/featurebase/v3/idk/mds" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/pql" - "github.com/molecula/featurebase/v3/prometheus" proto "github.com/molecula/featurebase/v3/proto" - "github.com/molecula/featurebase/v3/stats" "github.com/pkg/errors" prom "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -55,7 +53,6 @@ const ( ) // TODO Jaeger -// TODO Prometheus // Main holds all config for general ingest type Main struct { @@ -132,7 +129,6 @@ type Main struct { newNexter func(c int) (IDAllocator, error) ra RangeAllocator - stats stats.StatsClient metricsServer *http.Server log logger.Logger @@ -218,7 +214,7 @@ func NewMain() *Main { Concurrency: 1, CacheLength: 64, PackBools: "bools", - Namespace: "ingester", + Namespace: "ingester", // this is now ignored and hardcoded in metrics.go IDAllocKeyPrefix: "ingest", UseShardTransactionalEndpoint: os.Getenv("IDK_DEFAULT_SHARD_TRANSACTIONAL") != "", @@ -228,8 +224,6 @@ func NewMain() *Main { SchemaManager: NopSchemaManager, - stats: stats.NopStatsClient, - log: logger.NewStandardLogger(os.Stderr), } } @@ -457,7 +451,6 @@ initialFetch: if v, ok := source.(Metadata); ok { m.log.Printf("new schema - subject: %#v; version: %d; schema: %#v", v.SchemaSubject(), v.SchemaVersion(), v.SchemaSchema()) - // m.log.Printf("new schema: %#v", v.SchemaMetadata()) } else { m.log.Printf("new schema: %#v", schema) } @@ -465,7 +458,7 @@ initialFetch: if err != nil { return errors.Wrap(err, "batchFromSchema") } - m.stats.Count(MetricIngesterSchemaChanges, 1, 1) + CounterIngesterSchemaChanges.Inc() csvSlice = make([]string, len(schema)) if m.csvWriter != nil { for i := range schema { @@ -571,7 +564,7 @@ initialFetch: // skip bad rows only if !rowHasError { err = batch.Add(*row) - m.stats.Count(MetricIngesterRowsAdded, 1, 1) + CounterIngesterRowsAdded.Inc() } if err == pilosabatch.ErrBatchNowFull || err == pilosabatch.ErrBatchNowStale { @@ -958,7 +951,7 @@ func (m *Main) commitRecord(ctx context.Context, rec Record, limitCounter *msgCo return errors.Wrap(err, "committing") } limitCounter.Increment(numRecords) - m.stats.Count(MetricCommittedRecords, int64(numRecords), 1) + CounterCommittedRecords.Add(float64(numRecords)) return nil } @@ -972,7 +965,7 @@ func (m *Main) NewLookupClient() (*PostgresClient, error) { func (m *Main) setupClient() (*tls.Config, error) { var tlsConfig *tls.Config var err error - var opts = []pilosaclient.ClientOption{pilosaclient.OptClientStatsClient(m.stats)} + var opts = []pilosaclient.ClientOption{} if m.TLS.CertificatePath != "" { tlsConfig, err = GetTLSConfig(&m.TLS, m.Log()) if err != nil { @@ -1037,34 +1030,24 @@ func (m *Main) setupClient() (*tls.Config, error) { } func (m *Main) setupStats() error { - if m.Stats != "" { - opts := []prometheus.ClientOption{prometheus.OptClientNamespace(m.Namespace)} - m.stats, _ = prometheus.NewPrometheusClient(opts...) // ignore error that must be nil - - mux := http.NewServeMux() - // reg := prom.NewRegistry() // TODO switch to this once pilosa PrometheusClient is fixed and doesn't use the global registry internally. - // also change prom.DefaultGatherer to be "reg" at that time - reg := prom.DefaultRegisterer - promHandler := promhttp.InstrumentMetricHandler(reg, promhttp.HandlerFor(prom.DefaultGatherer, promhttp.HandlerOpts{})) - mux.Handle("/metrics", promHandler) - - mux.Handle("/metrics.json", metricsJSONHandler{metricsURI: "http://" + m.Stats + "/metrics"}) - m.metricsServer = &http.Server{Addr: m.Stats, Handler: mux} - ln, err := net.Listen("tcp", m.Stats) - if err != nil { - return errors.Wrapf(err, "listen for metrics on '%s'", m.Stats) - } - - go func() { - m.log.Printf("Serving Prometheus metrics with namespace \"%s\" at %v/metrics\n", m.Namespace, m.Stats) - err = m.metricsServer.Serve(ln) - if err != http.ErrServerClosed { - m.log.Printf("serve metrics on '%s': %v", m.Stats, err) - } - }() + mux := http.NewServeMux() + promHandler := promhttp.InstrumentMetricHandler(prom.DefaultRegisterer, promhttp.HandlerFor(prom.DefaultGatherer, promhttp.HandlerOpts{})) + mux.Handle("/metrics", promHandler) + mux.Handle("/metrics.json", metricsJSONHandler{metricsURI: "http://" + m.Stats + "/metrics"}) + m.metricsServer = &http.Server{Addr: m.Stats, Handler: mux} + ln, err := net.Listen("tcp", m.Stats) + if err != nil { + return errors.Wrapf(err, "listen for metrics on '%s'", m.Stats) } - return nil + go func() { + m.log.Printf("Serving Prometheus metrics with namespace \"%s\" at %v/metrics\n", m.Namespace, m.Stats) + err = m.metricsServer.Serve(ln) + if err != http.ErrServerClosed { + m.log.Printf("serve metrics on '%s': %v", m.Stats, err) + } + }() + return nil } type metricsJSONHandler struct { @@ -1218,7 +1201,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error { if err != nil { return errors.Wrap(err, "clearing bools") } - m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:packed-bool"}) + CounterDeleterRowsAdded.With(prom.Labels{"type": "packed-bool"}).Inc() continue } else { fieldName = directive @@ -1267,7 +1250,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error { if err != nil { return errors.Wrap(err, "clearing set") } - m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:set"}) + CounterDeleterRowsAdded.With(prom.Labels{"type": "set"}).Inc() case pilosaclient.FieldTypeMutex: if val == "" { continue @@ -1278,7 +1261,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error { if err != nil { return errors.Wrap(err, "clearing mutex") } - m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:mutex"}) + CounterDeleterRowsAdded.With(prom.Labels{"type": "mutex"}).Inc() case pilosaclient.FieldTypeBool: _, err := client.Query(index.BatchQuery( field.Clear(0, recordID), @@ -1287,19 +1270,19 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error { if err != nil { return errors.Wrap(err, "clearing bool") } - m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:bool"}) + CounterDeleterRowsAdded.With(prom.Labels{"type": "bool"}).Inc() case pilosaclient.FieldTypeInt: _, err := client.Query(field.Clear(0, recordID)) if err != nil { return errors.Wrap(err, "clearing int") } - m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:int"}) + CounterDeleterRowsAdded.With(prom.Labels{"type": "int"}).Inc() case pilosaclient.FieldTypeDecimal: _, err := client.Query(field.Clear(0, recordID)) if err != nil { return errors.Wrap(err, "clearing decimal") } - m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:decimal"}) + CounterDeleterRowsAdded.With(prom.Labels{"type": "decimal"}).Inc() case pilosaclient.FieldTypeTime: return errors.Errorf("deletion on time fields unimplemented") default: diff --git a/idk/ingest_test.go b/idk/ingest_test.go index 5c6dd805b..f5ae6b059 100644 --- a/idk/ingest_test.go +++ b/idk/ingest_test.go @@ -525,7 +525,7 @@ func TestIngesterServesPrometheusEndpoint(t *testing.T) { if err != nil { t.Errorf("read error: %v", err) } - if strings.Contains(string(contents), MetricIngesterRowsAdded) { + if !strings.Contains(string(contents), MetricIngesterRowsAdded) { t.Errorf("metric name missing: %v", MetricIngesterRowsAdded) } close(records) diff --git a/idk/mds/importer.go b/idk/mds/importer.go index 25833cef6..d6bf7de35 100644 --- a/idk/mds/importer.go +++ b/idk/mds/importer.go @@ -248,8 +248,6 @@ func (m *importer) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field return fbClient.DoImport(string(qtbl.Key()), shard, path, data) } -func (m *importer) StatsTiming(name string, value time.Duration, rate float64) {} - // getQtbl takes a table (TableKey) and sets the local m.qtbl value. When we // originally set up this type, it was only used by IDK, and the table was known // at the beginning of the process, so it could be set on this import. But diff --git a/idk/metrics.go b/idk/metrics.go index e7d0feabe..cdaac3b33 100644 --- a/idk/metrics.go +++ b/idk/metrics.go @@ -1,8 +1,52 @@ package idk +import "github.com/prometheus/client_golang/prometheus" + const ( MetricDeleterRowsAdded = "deleter_rows_added_total" MetricIngesterRowsAdded = "ingester_rows_added_total" MetricIngesterSchemaChanges = "ingester_schema_changes_total" MetricCommittedRecords = "committed_records" ) + +var CounterIngesterSchemaChanges = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "ingester", + Name: MetricIngesterSchemaChanges, + Help: "TODO", + }, +) + +var CounterIngesterRowsAdded = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "ingester", + Name: MetricIngesterRowsAdded, + Help: "TODO", + }, +) + +var CounterCommittedRecords = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "ingester", + Name: MetricCommittedRecords, + Help: "TODO", + }, +) + +var CounterDeleterRowsAdded = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "ingester", + Name: MetricDeleterRowsAdded, + Help: "TODO", + }, + []string{ + "type", + }, +) + +func init() { + prometheus.MustRegister(CounterIngesterSchemaChanges) + prometheus.MustRegister(CounterIngesterRowsAdded) + prometheus.MustRegister(CounterCommittedRecords) + prometheus.MustRegister(CounterDeleterRowsAdded) +} diff --git a/importer.go b/importer.go index 63e62e5b0..e9a8ad7b6 100644 --- a/importer.go +++ b/importer.go @@ -20,8 +20,6 @@ type Importer interface { EncodeImportValues(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, vals []int64, ids []uint64, clear bool) (path string, data []byte, err error) EncodeImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, vals, ids []uint64, clear bool) (path string, data []byte, err error) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, path string, data []byte) error - - StatsTiming(name string, value time.Duration, rate float64) } // Ensure type implements interface. diff --git a/index.go b/index.go index d3d8ee962..26e7a3ee5 100644 --- a/index.go +++ b/index.go @@ -16,9 +16,9 @@ import ( "github.com/molecula/featurebase/v3/disco" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/roaring" - "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/testhook" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "golang.org/x/sync/errgroup" ) @@ -43,7 +43,6 @@ type Index struct { broadcaster broadcaster serializer Serializer - Stats stats.StatsClient // Passed to field for foreign-index lookup. holder *Holder @@ -82,7 +81,6 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) { fields: make(map[string]*Field), broadcaster: NopBroadcaster, - Stats: stats.NopStatsClient, holder: holder, trackExistence: true, @@ -510,7 +508,7 @@ func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap { b.UnionInPlace(f.AvailableShards(localOnly)) } - i.Stats.Gauge(MetricMaxShard, float64(b.Max()), 1.0) + GaugeIndexMaxShard.With(prometheus.Labels{"index": i.name}).Set(float64(b.Max())) return b } @@ -923,7 +921,6 @@ func (i *Index) newField(path, name string) (*Field, error) { return nil, err } f.idx = i - f.Stats = i.Stats f.broadcaster = i.broadcaster f.serializer = i.serializer f.OpenTranslateStore = i.OpenTranslateStore diff --git a/metrics.go b/metrics.go index 9c95eb046..84bfbb591 100644 --- a/metrics.go +++ b/metrics.go @@ -1,6 +1,8 @@ // Copyright 2021 Molecula Corp. All rights reserved. package pilosa +import "github.com/prometheus/client_golang/prometheus" + const ( MetricCreateIndex = "create_index_total" MetricDeleteIndex = "delete_index_total" @@ -52,3 +54,997 @@ const ( MetricSqlQueries = "sql_queries_total" MetricDeleteDataframe = "delete_dataframe" ) + +const ( + // MetricBatchImportDurationSeconds records the full time of the + // RecordBatch.Import call. This includes starting and finishing a + // transaction, doing key translation, building fragments locally, + // importing all data, and resetting internal structures. + MetricBatchImportDurationSeconds = "batch_import_duration_seconds" + + // MetricBatchFlushDurationSeconds records the full time for + // RecordBatch.Flush (if splitBatchMode is in use). This includes + // starting and finishing a transaction, importing all data, and + // resetting internal structures. + MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds" + + // MetricBatchShardImportBuildRequestsSeconds is the time it takes + // after making fragments to build the shard-transactional request + // objects (but not actually import them or do any network activity). + MetricBatchShardImportBuildRequestsSeconds = "batch_shard_import_build_requests_seconds" + + // MetricBatchShardImportDurationSeconds is the time it takes to + // import all data for all shards in the batch using the + // shard-transactional endpoint. This does not include the time it + // takes to build the requests locally. + MetricBatchShardImportDurationSeconds = "batch_shard_import_duration_seconds" +) + +// server related + +var CounterJobTotal = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "job_total", + Help: "TODO", + }, +) + +var GaugeWorkerTotal = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: "worker_total", + Help: "TODO", + }, +) + +var CounterPQLQueries = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricPqlQueries, + Help: "TODO", + }, +) + +var CounterSQLQueries = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricSqlQueries, + Help: "TODO", + }, +) + +var CounterGarbageCollection = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricGarbageCollection, + Help: "TODO", + }, +) + +var GaugeGoroutines = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricGoroutines, + Help: "TODO", + }, +) + +var GaugeOpenFiles = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricOpenFiles, + Help: "TODO", + }, +) + +var GaugeHeapAlloc = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricHeapAlloc, + Help: "TODO", + }, +) + +var GaugeHeapInUse = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricHeapInuse, + Help: "TODO", + }, +) + +var GaugeStackInUse = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricStackInuse, + Help: "TODO", + }, +) + +var GaugeMallocs = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricMallocs, + Help: "TODO", + }, +) + +var GaugeFrees = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricFrees, + Help: "TODO", + }, +) + +var SummaryHttpRequests = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricHTTPRequest, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, + []string{ + "method", + "path", + "slow", + "useragent", + "where", + }, +) + +var CounterCreateIndex = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricCreateIndex, + Help: "TODO", + }, +) + +var CounterDeleteIndex = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricDeleteIndex, + Help: "TODO", + }, +) + +var CounterDeleteDataframe = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricDeleteDataframe, + Help: "TODO", + }, +) + +var CounterCreateField = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricCreateField, + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterDeleteField = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricDeleteField, + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterDeleteAvailableShard = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricDeleteAvailableShard, + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterExclusiveTransactionRequest = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricExclusiveTransactionRequest, + Help: "TODO", + }, +) + +var CounterTransactionStart = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricTransactionStart, + Help: "TODO", + }, +) + +var CounterExclusiveTransactionBlocked = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricExclusiveTransactionBlocked, + Help: "TODO", + }, +) + +var CounterTransactionBlocked = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricTransactionBlocked, + Help: "TODO", + }, +) + +var CounterExclusiveTransactionActive = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricExclusiveTransactionActive, + Help: "TODO", + }, +) + +var CounterExclusiveTransactionEnd = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricExclusiveTransactionEnd, + Help: "TODO", + }, +) + +var CounterTransactionEnd = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricTransactionEnd, + Help: "TODO", + }, +) + +// TODO(pok) do these need index names? +var CounterRecalculateCache = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricRecalculateCache, + Help: "TODO", + }, +) + +var CounterInvalidateCacheSkipped = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricInvalidateCacheSkipped, + Help: "TODO", + }, +) + +var CounterInvalidateCache = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricInvalidateCache, + Help: "TODO", + }, +) + +var GaugeRankCacheLength = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricRankCacheLength, + Help: "TODO", + }, +) + +var CounterCacheThresholdReached = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricCacheThresholdReached, + Help: "TODO", + }, +) + +var CounterReadDirtyCache = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricReadDirtyCache, + Help: "TODO", + }, +) + +var CounterSetBit = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricSetBit, + Help: "TODO", + }, +) + +var CounterClearBit = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricClearBit, + Help: "TODO", + }, +) + +var CounterSetRow = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "setRow", + Help: "TODO", + }, +) + +var CounterImportingN = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricImportingN, + Help: "TODO", + }, +) + +var CounterImportedN = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricImportedN, + Help: "TODO", + }, +) + +var CounterClearingingN = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricClearingN, + Help: "TODO", + }, +) + +var CounterClearedN = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: MetricClearedN, + Help: "TODO", + }, +) + +var SummaryGRPCStreamQueryDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricGRPCStreamQueryDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryGRPCStreamFormatDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricGRPCStreamFormatDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryGRPCUnaryQueryDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricGRPCUnaryQueryDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryGRPCUnaryFormatDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricGRPCUnaryFormatDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryBatchImportDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricBatchImportDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryBatchFlushDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricBatchFlushDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryBatchShardImportBuildRequestsSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricBatchShardImportBuildRequestsSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +var SummaryBatchShardImportDurationSeconds = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "pilosa", + Name: MetricBatchShardImportDurationSeconds, + Help: "TODO", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, +) + +// index pql call related + +var CounterQuerySumTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_sum_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryMinTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_min_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryMaxTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_max_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryMinRowTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_minrow_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryMaxRowTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_maxrow_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryClearTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_clear_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryClearRowTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_clearrow_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryDistinctTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_distinct_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryStoreTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_store_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryCountTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_count_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQuerySetTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_set_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryTopKTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_topk_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryTopNTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_topn_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryRowsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_rows_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryExternalLookupTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_externallookup_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryExtractTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_extract_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryGroupByTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_groupby_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryOptionsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_options_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryIncludesColumnTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_includescolumn_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryFieldValueTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_fieldvalue_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryPrecomputedTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_precomputed_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryUnionRowsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_unionrows_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryConstRowTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_constrow_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryLimitTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_limit_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryPercentileTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_percentile_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryDeleteTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_delete_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQuerySortTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_sort_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryApplyTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_apply_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryArrowTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_arrow_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +// bitmap calls +var CounterQueryBitmapTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_bitmap_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryRowTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_row_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryRowBSITotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_row_bsi_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryRangeTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_range_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryDifferenceTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_difference_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryIntersectTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_intersect_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryUnionTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_union_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryInnerUnionRowsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_innerunionrows_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryXorTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_xor_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryNotTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_not_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryShiftTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_shift_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +var CounterQueryAllTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pilosa", + Name: "query_all_total", + Help: "TODO", + }, + []string{ + "index", + }, +) + +// index related + +var GaugeIndexMaxShard = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pilosa", + Name: MetricMaxShard, + Help: "TODO", + }, + []string{ + "index", + }, +) + +func init() { + // server related + prometheus.MustRegister(CounterJobTotal) + prometheus.MustRegister(GaugeWorkerTotal) + prometheus.MustRegister(CounterPQLQueries) + prometheus.MustRegister(CounterSQLQueries) + prometheus.MustRegister(CounterGarbageCollection) + prometheus.MustRegister(GaugeGoroutines) + prometheus.MustRegister(GaugeOpenFiles) + prometheus.MustRegister(GaugeHeapAlloc) + prometheus.MustRegister(GaugeHeapInUse) + prometheus.MustRegister(GaugeStackInUse) + prometheus.MustRegister(GaugeMallocs) + prometheus.MustRegister(GaugeFrees) + prometheus.MustRegister(SummaryHttpRequests) + prometheus.MustRegister(CounterCreateIndex) + prometheus.MustRegister(CounterDeleteIndex) + prometheus.MustRegister(CounterCreateField) + prometheus.MustRegister(CounterDeleteField) + prometheus.MustRegister(CounterDeleteAvailableShard) + prometheus.MustRegister(CounterDeleteDataframe) + prometheus.MustRegister(CounterExclusiveTransactionRequest) + prometheus.MustRegister(CounterTransactionStart) + prometheus.MustRegister(CounterExclusiveTransactionBlocked) + prometheus.MustRegister(CounterTransactionBlocked) + prometheus.MustRegister(CounterExclusiveTransactionActive) + prometheus.MustRegister(CounterExclusiveTransactionEnd) + prometheus.MustRegister(CounterTransactionEnd) + prometheus.MustRegister(CounterRecalculateCache) + prometheus.MustRegister(CounterInvalidateCacheSkipped) + prometheus.MustRegister(CounterInvalidateCache) + prometheus.MustRegister(GaugeRankCacheLength) + prometheus.MustRegister(CounterCacheThresholdReached) + prometheus.MustRegister(CounterReadDirtyCache) + prometheus.MustRegister(CounterSetBit) + prometheus.MustRegister(CounterClearBit) + prometheus.MustRegister(CounterSetRow) + prometheus.MustRegister(CounterImportingN) + prometheus.MustRegister(CounterImportedN) + prometheus.MustRegister(CounterClearingingN) + prometheus.MustRegister(CounterClearedN) + prometheus.MustRegister(SummaryGRPCStreamQueryDurationSeconds) + prometheus.MustRegister(SummaryGRPCStreamFormatDurationSeconds) + prometheus.MustRegister(SummaryGRPCUnaryQueryDurationSeconds) + prometheus.MustRegister(SummaryGRPCUnaryFormatDurationSeconds) + prometheus.MustRegister(SummaryBatchImportDurationSeconds) + prometheus.MustRegister(SummaryBatchFlushDurationSeconds) + prometheus.MustRegister(SummaryBatchShardImportBuildRequestsSeconds) + prometheus.MustRegister(SummaryBatchShardImportDurationSeconds) + + // pql calls + prometheus.MustRegister(CounterQuerySumTotal) + prometheus.MustRegister(CounterQueryMinTotal) + prometheus.MustRegister(CounterQueryMaxTotal) + prometheus.MustRegister(CounterQueryMinRowTotal) + prometheus.MustRegister(CounterQueryMaxRowTotal) + prometheus.MustRegister(CounterQueryClearTotal) + prometheus.MustRegister(CounterQueryClearRowTotal) + prometheus.MustRegister(CounterQueryDistinctTotal) + prometheus.MustRegister(CounterQueryStoreTotal) + prometheus.MustRegister(CounterQueryCountTotal) + prometheus.MustRegister(CounterQuerySetTotal) + prometheus.MustRegister(CounterQueryTopKTotal) + prometheus.MustRegister(CounterQueryTopNTotal) + prometheus.MustRegister(CounterQueryRowsTotal) + prometheus.MustRegister(CounterQueryExternalLookupTotal) + prometheus.MustRegister(CounterQueryExtractTotal) + prometheus.MustRegister(CounterQueryGroupByTotal) + prometheus.MustRegister(CounterQueryOptionsTotal) + prometheus.MustRegister(CounterQueryIncludesColumnTotal) + prometheus.MustRegister(CounterQueryFieldValueTotal) + prometheus.MustRegister(CounterQueryPrecomputedTotal) + prometheus.MustRegister(CounterQueryUnionRowsTotal) + prometheus.MustRegister(CounterQueryConstRowTotal) + prometheus.MustRegister(CounterQueryLimitTotal) + prometheus.MustRegister(CounterQueryPercentileTotal) + prometheus.MustRegister(CounterQueryDeleteTotal) + prometheus.MustRegister(CounterQuerySortTotal) + prometheus.MustRegister(CounterQueryApplyTotal) + prometheus.MustRegister(CounterQueryArrowTotal) + prometheus.MustRegister(CounterQueryBitmapTotal) + prometheus.MustRegister(CounterQueryRowTotal) + prometheus.MustRegister(CounterQueryRowBSITotal) + prometheus.MustRegister(CounterQueryRangeTotal) + prometheus.MustRegister(CounterQueryDifferenceTotal) + prometheus.MustRegister(CounterQueryIntersectTotal) + prometheus.MustRegister(CounterQueryUnionTotal) + prometheus.MustRegister(CounterQueryInnerUnionRowsTotal) + prometheus.MustRegister(CounterQueryXorTotal) + prometheus.MustRegister(CounterQueryNotTotal) + prometheus.MustRegister(CounterQueryShiftTotal) + prometheus.MustRegister(CounterQueryAllTotal) + + // index related + prometheus.MustRegister(GaugeIndexMaxShard) + +} diff --git a/performancecounters.go b/performancecounters.go new file mode 100644 index 000000000..68fcad729 --- /dev/null +++ b/performancecounters.go @@ -0,0 +1,203 @@ +package pilosa + +import ( + "sync/atomic" + + "github.com/prometheus/client_golang/prometheus" +) + +// PerformanceCounter holds data about a performance counter for external consumers +type PerformanceCounter struct { + NameSpace string + SubSystem string + CounterName string + Help string + Value int64 + CounterType int64 +} + +// constants for the counter types +const ( + // raw - for when you just want a count of something + CTR_TYPE_RAW = 0 + // per second - for when you accumulate counts of things + // a consumer would sample this at intervals to arrive at a delta + // then divide by the time in seconds between the samples to get a + // per-second value + CTR_TYPE_PER_SECOND = 1 + // ratio - for when you accumulate a count of something that you + // want to use as a numerator in a ratio calculation + // e.g. 'cache hits' could be a counter of this type and you could + // divide it by a 'cache lookups' counter to get the hit ratio (see below) + CTR_TYPE_RATIO = 2 + // ratio base - for when you accumulate a count of something that you + // want to use as a denominator in a ratio calculation + // e.g. 'cache lookups' could be a counter of this type and you could + // use it as the denominator in a division with a 'cache hits' counter + // as the numerator to get the hit ratio + CTR_TYPE_RATIO_BASE = 3 +) + +type PerformanceCounters struct { + counterValues [5]perfCtrWrapper +} + +type perfCtr struct { + nameSpace string + subSystem string + name string + help string + value int64 + counterType int64 +} + +func (p *perfCtr) Add(increment int64) { + atomic.AddInt64(&p.value, increment) +} + +type perfCtrWrapper struct { + ctr *perfCtr + fun prometheus.CounterFunc +} + +var PerfCounterSQLRequestSec = perfCtr{ + nameSpace: "pilosa", + subSystem: "sql_statistics", + name: "sql_requests_sec", + help: "TODO", + value: 0, + counterType: CTR_TYPE_PER_SECOND, +} + +var PerfCounterSQLInsertsSec = perfCtr{ + nameSpace: "pilosa", + subSystem: "sql_statistics", + name: "sql_inserts_sec", + help: "TODO", + value: 0, + counterType: CTR_TYPE_PER_SECOND, +} + +var PerfCounterSQLBulkInsertsSec = perfCtr{ + nameSpace: "pilosa", + subSystem: "sql_statistics", + name: "sql_bulk_inserts_sec", + help: "TODO", + value: 0, + counterType: CTR_TYPE_PER_SECOND, +} + +var PerfCounterSQLBulkInsertBatchesSec = perfCtr{ + nameSpace: "pilosa", + subSystem: "sql_statistics", + name: "sql_bulk_insert_batches_sec", + help: "TODO", + value: 0, + counterType: CTR_TYPE_PER_SECOND, +} + +var PerfCounterSQLDeletesSec = perfCtr{ + nameSpace: "pilosa", + subSystem: "sql_statistics", + name: "sql_deletes_sec", + help: "TODO", + value: 0, + counterType: CTR_TYPE_PER_SECOND, +} + +var PerfCounters *PerformanceCounters = newPerformanceCounters() + +func newPerformanceCounters() *PerformanceCounters { + ctrs := &PerformanceCounters{ + counterValues: [5]perfCtrWrapper{ + { + &PerfCounterSQLRequestSec, + prometheus.NewCounterFunc( + prometheus.CounterOpts{ + Namespace: PerfCounterSQLRequestSec.nameSpace, + Subsystem: PerfCounterSQLRequestSec.subSystem, + Name: PerfCounterSQLRequestSec.name, + Help: PerfCounterSQLRequestSec.help, + }, + func() float64 { + return float64(atomic.LoadInt64(&PerfCounterSQLRequestSec.value)) + }), + }, + { + &PerfCounterSQLInsertsSec, + prometheus.NewCounterFunc( + prometheus.CounterOpts{ + Namespace: PerfCounterSQLInsertsSec.nameSpace, + Subsystem: PerfCounterSQLInsertsSec.subSystem, + Name: PerfCounterSQLInsertsSec.name, + Help: PerfCounterSQLInsertsSec.help, + }, + func() float64 { + return float64(atomic.LoadInt64(&PerfCounterSQLInsertsSec.value)) + }), + }, + { + &PerfCounterSQLBulkInsertsSec, + prometheus.NewCounterFunc( + prometheus.CounterOpts{ + Namespace: PerfCounterSQLBulkInsertsSec.nameSpace, + Subsystem: PerfCounterSQLBulkInsertsSec.subSystem, + Name: PerfCounterSQLBulkInsertsSec.name, + Help: PerfCounterSQLBulkInsertsSec.help, + }, + func() float64 { + return float64(atomic.LoadInt64(&PerfCounterSQLBulkInsertsSec.value)) + }), + }, + { + &PerfCounterSQLBulkInsertBatchesSec, + prometheus.NewCounterFunc( + prometheus.CounterOpts{ + Namespace: PerfCounterSQLBulkInsertBatchesSec.nameSpace, + Subsystem: PerfCounterSQLBulkInsertBatchesSec.subSystem, + Name: PerfCounterSQLBulkInsertBatchesSec.name, + Help: PerfCounterSQLBulkInsertBatchesSec.help, + }, + func() float64 { + return float64(atomic.LoadInt64(&PerfCounterSQLBulkInsertBatchesSec.value)) + }), + }, + { + &PerfCounterSQLDeletesSec, + prometheus.NewCounterFunc( + prometheus.CounterOpts{ + Namespace: PerfCounterSQLDeletesSec.nameSpace, + Subsystem: PerfCounterSQLDeletesSec.subSystem, + Name: PerfCounterSQLDeletesSec.name, + Help: PerfCounterSQLDeletesSec.help, + }, + func() float64 { + return float64(atomic.LoadInt64(&PerfCounterSQLDeletesSec.value)) + }), + }, + }, + } + + for _, w := range ctrs.counterValues { + prometheus.MustRegister(w.fun) + } + + return ctrs +} + +// list all the counters +// we can just read here without locking because if the counters get changed +// midway thru the loop, absent evidence to the contrary, the world will not end +func (p *PerformanceCounters) ListCounters() ([]PerformanceCounter, error) { + result := make([]PerformanceCounter, len(p.counterValues)) + for i, c := range p.counterValues { + result[i] = PerformanceCounter{ + NameSpace: c.ctr.nameSpace, + SubSystem: c.ctr.subSystem, + CounterName: c.ctr.name, + Value: c.ctr.value, + CounterType: c.ctr.counterType, + } + } + return result, nil +} diff --git a/prometheus/prometheus.go b/prometheus/prometheus.go deleted file mode 100644 index 79703cf85..000000000 --- a/prometheus/prometheus.go +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package prometheus - -import ( - "sort" - "strings" - "sync" - "time" - - "github.com/molecula/featurebase/v3/logger" - "github.com/molecula/featurebase/v3/stats" - "github.com/prometheus/client_golang/prometheus" -) - -const ( - // namespace is prepended to each metric event name with "_" - defaultNamespace = "general" -) - -// Ensure client implements interface. -var _ stats.StatsClient = &prometheusClient{} - -// Module-level mutex to avoid copying in WithTags() -var mu sync.Mutex - -// prometheusClient represents a Prometheus implementation of pilosa.statsClient. -type prometheusClient struct { - tags []string - logger logger.Logger - counters map[string]prometheus.Counter - counterVecs map[string]*prometheus.CounterVec - gauges map[string]prometheus.Gauge - gaugeVecs map[string]*prometheus.GaugeVec - observers map[string]prometheus.Observer - summaryVecs map[string]*prometheus.SummaryVec - namespace string -} - -// ClientOption is a functional option type for prometheusClient -type ClientOption func(c *prometheusClient) - -// OptClientPrefix is a functional option on prometheusClient used to set the namespace -func OptClientNamespace(namespace string) ClientOption { - return func(c *prometheusClient) { - c.namespace = namespace - } -} - -// NewPrometheusClient returns a new instance of StatsClient. -func NewPrometheusClient(opts ...ClientOption) (*prometheusClient, error) { - client := &prometheusClient{ - logger: logger.NopLogger, - counters: make(map[string]prometheus.Counter), - counterVecs: make(map[string]*prometheus.CounterVec), - gauges: make(map[string]prometheus.Gauge), - gaugeVecs: make(map[string]*prometheus.GaugeVec), - observers: make(map[string]prometheus.Observer), - summaryVecs: make(map[string]*prometheus.SummaryVec), - namespace: defaultNamespace, - } - - for _, opt := range opts { - opt(client) - } - - return client, nil -} - -// Open no-op to satisfy interface -func (c *prometheusClient) Open() {} - -// Close no-op to satisfy interface -func (c *prometheusClient) Close() error { - return nil -} - -// Tags returns a sorted list of tags on the client. -func (c *prometheusClient) Tags() []string { - return c.tags -} - -// labels returns an instance of prometheus.Labels with the value of the set tags. -func (c *prometheusClient) labels() prometheus.Labels { - return tagsToLabels(c.tags, c.logger) -} - -// WithTags returns a new client with additional tags appended. -func (c *prometheusClient) WithTags(tags ...string) stats.StatsClient { - return &prometheusClient{ - tags: unionStringSlice(c.tags, tags), - logger: c.logger, - counters: c.counters, - counterVecs: c.counterVecs, - gauges: c.gauges, - gaugeVecs: c.gaugeVecs, - observers: c.observers, - summaryVecs: c.summaryVecs, - namespace: c.namespace, - } -} - -// Count tracks the number of times something occurs per second. -func (c *prometheusClient) Count(name string, value int64, rate float64) { - mu.Lock() - defer mu.Unlock() - - var counter prometheus.Counter - var ok bool - name = strings.Replace(name, ".", "_", -1) - labels := c.labels() - opts := prometheus.CounterOpts{ - Namespace: c.namespace, - Name: name, - } - if len(labels) == 0 { - counter, ok = c.counters[name] - if !ok { - counter = prometheus.NewCounter(opts) - c.counters[name] = counter - prometheus.MustRegister(counter) - } - } else { - var counterVec *prometheus.CounterVec - counterVec, ok = c.counterVecs[name] - if !ok { - counterVec = prometheus.NewCounterVec( - opts, - labelKeys(labels), - ) - c.counterVecs[name] = counterVec - prometheus.MustRegister(counterVec) - } - var err error - counter, err = counterVec.GetMetricWith(labels) - if err != nil { - c.logger.Errorf("counterVec.GetMetricWith error: %s", err) - } - } - if value == 1 { - counter.Inc() - } else { - counter.Add(float64(value)) - } -} - -// CountWithCustomTags tracks the number of times something occurs per second with custom tags. -func (c *prometheusClient) CountWithCustomTags(name string, value int64, rate float64, t []string) { - c.WithTags(append(c.tags, t...)...).Count(name, value, rate) -} - -// Gauge sets the value of a metric. -func (c *prometheusClient) Gauge(name string, value float64, rate float64) { - mu.Lock() - defer mu.Unlock() - - var gauge prometheus.Gauge - var ok bool - name = strings.Replace(name, ".", "_", -1) - labels := c.labels() - opts := prometheus.GaugeOpts{ - Namespace: c.namespace, - Name: name, - } - if len(labels) == 0 { - gauge, ok = c.gauges[name] - if !ok { - gauge = prometheus.NewGauge(opts) - c.gauges[name] = gauge - prometheus.MustRegister(gauge) - } - } else { - var gaugeVec *prometheus.GaugeVec - gaugeVec, ok = c.gaugeVecs[name] - if !ok { - gaugeVec = prometheus.NewGaugeVec( - opts, - labelKeys(labels), - ) - c.gaugeVecs[name] = gaugeVec - prometheus.MustRegister(gaugeVec) - } - var err error - gauge, err = gaugeVec.GetMetricWith(labels) - if err != nil { - c.logger.Errorf("gaugeVec.GetMetricWith error: %s", err) - return - } - } - gauge.Set(float64(value)) -} - -// Histogram tracks statistical distribution of a metric. -func (c *prometheusClient) Histogram(name string, value float64, rate float64) { - mu.Lock() - defer mu.Unlock() - - var observer prometheus.Observer - var ok bool - name = strings.Replace(name, ".", "_", -1) - labels := c.labels() - opts := prometheus.SummaryOpts{ - Namespace: c.namespace, - Name: name, - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - } - if len(labels) == 0 { - observer, ok = c.observers[name] - if !ok { - summary := prometheus.NewSummary(opts) - observer = summary - c.observers[name] = observer - prometheus.MustRegister(summary) - } - } else { - var summaryVec *prometheus.SummaryVec - summaryVec, ok = c.summaryVecs[name] - if !ok { - summaryVec = prometheus.NewSummaryVec( - opts, - labelKeys(labels), - ) - c.summaryVecs[name] = summaryVec - prometheus.MustRegister(summaryVec) - } - var err error - observer, err = summaryVec.GetMetricWith(labels) - if err != nil { - c.logger.Errorf("summaryVec.GetMetricWith error: %s", err) - return - } - } - observer.Observe(value) -} - -// Set tracks number of unique elements. -func (c *prometheusClient) Set(name string, value string, rate float64) { - c.logger.Infof("prometheusClient.Set unimplemented: %s=%s", name, value) -} - -// Timing tracks timing information for a metric. -func (c *prometheusClient) Timing(name string, value time.Duration, rate float64) { - c.Histogram(name, value.Seconds(), rate) -} - -// SetLogger sets the logger for client. -func (c *prometheusClient) SetLogger(logger logger.Logger) { - c.logger = logger -} - -// unionStringSlice returns a sorted set of tags which combine a & b. -func unionStringSlice(a, b []string) []string { - // Sort both sets first. - sort.Strings(a) - sort.Strings(b) - - // Find size of largest slice. - n := len(a) - if len(b) > n { - n = len(b) - } - - // Exit if both sets are empty. - if n == 0 { - return nil - } - - // Iterate over both in order and merge. - other := make([]string, 0, n) - for len(a) > 0 || len(b) > 0 { - if len(a) == 0 { - other, b = append(other, b[0]), b[1:] - } else if len(b) == 0 { - other, a = append(other, a[0]), a[1:] - } else if a[0] < b[0] { - other, a = append(other, a[0]), a[1:] - } else if b[0] < a[0] { - other, b = append(other, b[0]), b[1:] - } else { - other, a, b = append(other, a[0]), a[1:], b[1:] - } - } - return other -} - -func tagsToLabels(tags []string, logger logger.Logger) (labels prometheus.Labels) { - labels = make(prometheus.Labels) - for _, tag := range tags { - tagParts := strings.SplitAfterN(tag, ":", 2) - if len(tagParts) != 2 { - // only process tags in "key:value" form - logger.Errorf("invalid Prometheus label: %v\n", tag) - continue - } - labels[tagParts[0][0:len(tagParts[0])-1]] = tagParts[1] - } - return labels -} - -func labelKeys(labels prometheus.Labels) (keys []string) { - keys = make([]string, len(labels)) - i := 0 - for k := range labels { - keys[i] = k - i++ - } - return keys -} diff --git a/prometheus/prometheus_test.go b/prometheus/prometheus_test.go index 1dd64ca9d..b3c3b7279 100644 --- a/prometheus/prometheus_test.go +++ b/prometheus/prometheus_test.go @@ -2,62 +2,32 @@ package prometheus_test import ( - "reflect" "testing" - "time" - pilosaPrometheus "github.com/molecula/featurebase/v3/prometheus" + "github.com/molecula/featurebase/v3/test" "github.com/prometheus/client_golang/prometheus" io_prometheus_client "github.com/prometheus/client_model/go" ) -func TestPrometheusClient_WithTags(t *testing.T) { - // Create a new client. - c, err := pilosaPrometheus.NewPrometheusClient() - if err != nil { - t.Fatal(err) - } - defer c.Close() - - // Create a new client with additional tags. - c1 := c.WithTags("foo", "bar") - if tags := c1.Tags(); !reflect.DeepEqual(tags, []string{"bar", "foo"}) { - t.Fatalf("unexpected tags: %+v", tags) - } - - // Create a new client from the clone with more tags. - c2 := c1.WithTags("bar", "baz") - if tags := c2.Tags(); !reflect.DeepEqual(tags, []string{"bar", "baz", "foo"}) { - t.Fatalf("unexpected tags: %+v", tags) - } -} - func TestPrometheusClient_Methods(t *testing.T) { - // Create a new client. - c, err := pilosaPrometheus.NewPrometheusClient( - pilosaPrometheus.OptClientNamespace("testns"), - ) - if err != nil { - t.Fatal(err) - } + c := test.MustRunCluster(t, 1) defer c.Close() - dur, _ := time.ParseDuration("123us") - c.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"}) - c.Count("cc", 1, 1.0) - c.Gauge("gg", 10, 1.0) - c.Histogram("hh", 1, 1.0) - c.Timing("tt", dur, 1.0) - metricFams, err := prometheus.DefaultGatherer.Gather() if err != nil { t.Fatal(err) } - for _, metricName := range []string{"testns_ct", "testns_cc", "testns_gg", "testns_hh", "testns_tt"} { + for _, metricName := range []string{ + "pilosa_sql_statistics_sql_bulk_insert_batches_sec", + "pilosa_sql_statistics_sql_bulk_inserts_sec", + "pilosa_sql_statistics_sql_deletes_sec", + "pilosa_sql_statistics_sql_inserts_sec", + "pilosa_sql_statistics_sql_requests_sec", + } { if metricExists(metricName, metricFams) { continue } - t.Fatalf("Metric was not recorded: %s", metricName) + t.Fatalf("metric does not exist: %s", metricName) } } diff --git a/server.go b/server.go index 1e7f8cbee..0618939dd 100644 --- a/server.go +++ b/server.go @@ -4,6 +4,7 @@ package pilosa import ( "context" "fmt" + "io" "log" "os" "os/exec" @@ -25,7 +26,6 @@ import ( "github.com/molecula/featurebase/v3/sql3" "github.com/molecula/featurebase/v3/sql3/parser" planner_types "github.com/molecula/featurebase/v3/sql3/planner/types" - "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/storage" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -245,15 +245,6 @@ func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption { } } -// OptServerStatsClient is a functional option on Server -// used to specify the stats client. -func OptServerStatsClient(sc stats.StatsClient) ServerOption { - return func(s *Server) error { - s.holderConfig.StatsClient = sc - return nil - } -} - // OptServerDiagnosticsInterval is a functional option on Server // used to specify the duration between diagnostic checks. func OptServerDiagnosticsInterval(dur time.Duration) ServerOption { @@ -541,7 +532,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { return nil, err } s.holder = NewHolder(path, s.holderConfig) - s.holder.Stats.SetLogger(s.logger) cwd, err := os.Getwd() if err != nil { return nil, err @@ -557,9 +547,6 @@ func NewServer(opts ...ServerOption) (*Server, error) { s.cluster.sharder = s.sharder s.cluster.serverlessStorage = s.serverlessStorage - // Append the NodeID tag to stats. - s.holder.Stats = s.holder.Stats.WithTags(fmt.Sprintf("node_id:%s", s.nodeID)) - s.executor.Holder = s.holder s.holder.executor = s.executor s.executor.Cluster = s.cluster @@ -657,7 +644,6 @@ func (s *Server) Open() error { s.syncer.Node = node s.syncer.Cluster = s.cluster s.syncer.Closing = s.closing - s.syncer.Stats = s.holder.Stats.WithTags("component:HolderSyncer") // Start background process listening for translation // sync resets. @@ -1249,26 +1235,26 @@ func (s *Server) monitorRuntime() { return case <-s.gcNotifier.AfterGC(): // GC just ran. - s.holder.Stats.Count(MetricGarbageCollection, 1, 1.0) + CounterGarbageCollection.Inc() case <-ticker.C: } // Record the number of go routines. - s.holder.Stats.Gauge(MetricGoroutines, float64(runtime.NumGoroutine()), 1.0) + GaugeGoroutines.Set(float64(runtime.NumGoroutine())) openFiles, err := countOpenFiles() // Open File handles. if err == nil { - s.holder.Stats.Gauge(MetricOpenFiles, float64(openFiles), 1.0) + GaugeOpenFiles.Set(float64(openFiles)) } // Runtime memory metrics. runtime.ReadMemStats(&m) - s.holder.Stats.Gauge(MetricHeapAlloc, float64(m.HeapAlloc), 1.0) - s.holder.Stats.Gauge(MetricHeapInuse, float64(m.HeapInuse), 1.0) - s.holder.Stats.Gauge(MetricStackInuse, float64(m.StackInuse), 1.0) - s.holder.Stats.Gauge(MetricMallocs, float64(m.Mallocs), 1.0) - s.holder.Stats.Gauge(MetricFrees, float64(m.Frees), 1.0) + GaugeHeapAlloc.Set(float64(m.HeapAlloc)) + GaugeHeapInUse.Set(float64(m.HeapInuse)) + GaugeStackInUse.Set(float64(m.StackInuse)) + GaugeMallocs.Set(float64(m.Mallocs)) + GaugeFrees.Set(float64(m.Frees)) } } @@ -1412,6 +1398,10 @@ func (s *Server) CompileExecutionPlan(ctx context.Context, q string) (planner_ty return s.executionPlannerFn(s.executor, s.executor.client.api, q).CompilePlan(ctx, st) } +func (s *Server) RehydratePlanOperator(ctx context.Context, reader io.Reader) (planner_types.PlanOperator, error) { + return s.executionPlannerFn(s.executor, s.executor.client.api, "").RehydratePlanOp(ctx, reader) +} + // countOpenFiles on operating systems that support lsof. func countOpenFiles() (int, error) { switch runtime.GOOS { diff --git a/server/grpc.go b/server/grpc.go index d0c0532aa..e4e24451a 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -22,7 +22,6 @@ import ( pb "github.com/molecula/featurebase/v3/proto" vdsm_pb "github.com/molecula/featurebase/v3/proto/vdsm" "github.com/molecula/featurebase/v3/sql" - "github.com/molecula/featurebase/v3/stats" "github.com/pkg/errors" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -40,12 +39,11 @@ type GRPCHandler struct { perms *authz.GroupPermissions logger logger.Logger queryLogger logger.Logger - stats stats.StatsClient inspectDeprecated sync.Once } func NewGRPCHandler(api *pilosa.API) *GRPCHandler { - return &GRPCHandler{api: api, logger: logger.NopLogger, stats: stats.NopStatsClient} + return &GRPCHandler{api: api, logger: logger.NopLogger} } func (h *GRPCHandler) WithLogger(logger logger.Logger) *GRPCHandler { @@ -53,11 +51,6 @@ func (h *GRPCHandler) WithLogger(logger logger.Logger) *GRPCHandler { return h } -func (h *GRPCHandler) WithStats(stats stats.StatsClient) *GRPCHandler { - h.stats = stats - return h -} - func (h *GRPCHandler) WithPerms(perms *authz.GroupPermissions) *GRPCHandler { h.perms = perms return h @@ -139,7 +132,7 @@ func errToStatusError(err error) error { } func (h *GRPCHandler) execSQL(ctx context.Context, queryStr string) (pb.ToRowser, error) { - h.stats.Count(pilosa.MetricSqlQueries, 1, 1) + pilosa.CounterSQLQueries.Inc() return execSQL(ctx, h.api, h.logger, queryStr) } @@ -339,9 +332,9 @@ func (h *GRPCHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQ return errToStatusError(err) } durFormat := time.Since(t) - h.stats.Timing(pilosa.MetricGRPCStreamQueryDurationSeconds, durQuery, 0.1) - h.stats.Timing(pilosa.MetricGRPCStreamFormatDurationSeconds, durFormat, 0.1) - h.stats.Count(pilosa.MetricPqlQueries, 1, 1) + pilosa.SummaryGRPCStreamQueryDurationSeconds.Observe(durQuery.Seconds()) + pilosa.SummaryGRPCStreamFormatDurationSeconds.Observe(durFormat.Seconds()) + pilosa.CounterPQLQueries.Inc() return errToStatusError(nil) } @@ -405,9 +398,9 @@ func (h *GRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest return nil, errors.Wrap(err, "sending header") } - h.stats.Timing(pilosa.MetricGRPCUnaryQueryDurationSeconds, durQuery, 0.1) - h.stats.Timing(pilosa.MetricGRPCUnaryFormatDurationSeconds, durFormat, 0.1) - h.stats.Count(pilosa.MetricPqlQueries, 1, 1) + pilosa.SummaryGRPCUnaryQueryDurationSeconds.Observe(durQuery.Seconds()) + pilosa.SummaryGRPCUnaryFormatDurationSeconds.Observe(durFormat.Seconds()) + pilosa.CounterPQLQueries.Inc() return table, errToStatusError(nil) } @@ -509,11 +502,10 @@ type VDSMGRPCHandler struct { grpcHandler *GRPCHandler api *pilosa.API logger logger.Logger - stats stats.StatsClient } func NewVDSMGRPCHandler(grpcHandler *GRPCHandler, api *pilosa.API) *VDSMGRPCHandler { - return &VDSMGRPCHandler{grpcHandler: grpcHandler, api: api, logger: logger.NopLogger, stats: stats.NopStatsClient} + return &VDSMGRPCHandler{grpcHandler: grpcHandler, api: api, logger: logger.NopLogger} } func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler { @@ -521,11 +513,6 @@ func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler { return h } -func (h *VDSMGRPCHandler) WithStats(stats stats.StatsClient) *VDSMGRPCHandler { - h.stats = stats - return h -} - // GetVDSs returns a single VDS given a name func (h *VDSMGRPCHandler) GetVDS(ctx context.Context, req *vdsm_pb.GetVDSRequest) (*vdsm_pb.GetVDSResponse, error) { typedIdOrName := req.GetIdOrName() @@ -1480,7 +1467,6 @@ type grpcServer struct { logger logger.Logger queryLogger logger.Logger - stats stats.StatsClient } type grpcServerOption func(s *grpcServer) error @@ -1513,13 +1499,6 @@ func OptGRPCServerLogger(logger logger.Logger) grpcServerOption { } } -func OptGRPCServerStats(stats stats.StatsClient) grpcServerOption { - return func(s *grpcServer) error { - s.stats = stats - return nil - } -} - func OptGRPCServerAuth(authn *authn.Auth) grpcServerOption { return func(s *grpcServer) error { s.auth = authn @@ -1639,7 +1618,7 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) { // create grpc server server.grpcServer = grpc.NewServer(gopts...) - grpcHandler := NewGRPCHandler(server.api).WithLogger(server.logger).WithStats(server.stats).WithQueryLogger(server.queryLogger) + grpcHandler := NewGRPCHandler(server.api).WithLogger(server.logger).WithQueryLogger(server.queryLogger) // add server permissions if we've got 'em if server.perms != nil { @@ -1647,7 +1626,7 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) { } pb.RegisterPilosaServer(server.grpcServer, grpcHandler) - vdsm_pb.RegisterMoleculaServer(server.grpcServer, NewVDSMGRPCHandler(grpcHandler, server.api).WithLogger(server.logger).WithStats(server.stats)) + vdsm_pb.RegisterMoleculaServer(server.grpcServer, NewVDSMGRPCHandler(grpcHandler, server.api).WithLogger(server.logger)) // register the server so its services are available to grpc_cli and others reflection.Register(server.grpcServer) diff --git a/server/handler_test.go b/server/handler_test.go index 399750165..991d8da8b 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -1079,15 +1079,6 @@ func TestHandler_Endpoints(t *testing.T) { } }) - t.Run("Expvars", func(t *testing.T) { - w := httptest.NewRecorder() - r := test.MustNewHTTPRequest("GET", "/debug/vars", nil) - h.ServeHTTP(w, r) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } - }) - t.Run("Recalculate Caches", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil)) diff --git a/server/server.go b/server/server.go index 2847a12fb..011550dca 100644 --- a/server/server.go +++ b/server/server.go @@ -39,12 +39,9 @@ import ( "github.com/molecula/featurebase/v3/gopsutil" "github.com/molecula/featurebase/v3/logger" pnet "github.com/molecula/featurebase/v3/net" - "github.com/molecula/featurebase/v3/prometheus" "github.com/molecula/featurebase/v3/sql3" "github.com/molecula/featurebase/v3/sql3/planner" "github.com/molecula/featurebase/v3/statik" - "github.com/molecula/featurebase/v3/stats" - "github.com/molecula/featurebase/v3/statsd" "github.com/molecula/featurebase/v3/systemlayer" "github.com/molecula/featurebase/v3/syswrap" "github.com/molecula/featurebase/v3/testhook" @@ -474,11 +471,6 @@ func (m *Command) setupServer() error { diagnosticsInterval = defaultDiagnosticsInterval } - statsClient, err := newStatsClient(m.Config.Metric.Service, m.Config.Metric.Host, m.Config.Namespace()) - if err != nil { - return errors.Wrap(err, "new stats client") - } - if m.Config.Listener == nil { m.ln, err = getListener(*uri, m.tlsConfig) if err != nil { @@ -581,7 +573,6 @@ func (m *Command) setupServer() error { pilosa.OptServerQueryLogger(m.queryLogger), pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()), pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()), - pilosa.OptServerStatsClient(statsClient), pilosa.OptServerURI(advertiseURI), pilosa.OptServerGRPCURI(advertiseGRPCURI), pilosa.OptServerClusterName(m.Config.Cluster.Name), @@ -691,7 +682,6 @@ func (m *Command) setupServer() error { OptGRPCServerListener(m.grpcLn), OptGRPCServerTLSConfig(m.tlsConfig), OptGRPCServerLogger(m.logger), - OptGRPCServerStats(statsClient), OptGRPCServerAuth(m.auth), OptGRPCServerPerm(&p), OptGRPCServerQueryLogger(m.queryLogger), @@ -828,24 +818,6 @@ func (m *Command) Close() error { } } -// newStatsClient creates a stats client from the config -func newStatsClient(name string, host string, namespace string) (stats.StatsClient, error) { - switch name { - case "expvar": - return stats.NewExpvarStatsClient(), nil - case "statsd": - return statsd.NewStatsClient(host, namespace) - case "prometheus": - return prometheus.NewPrometheusClient( - prometheus.OptClientNamespace(namespace), - ) - case "nop", "none": - return stats.NopStatsClient, nil - default: - return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, prometheus, none].", name) - } -} - // getListener gets a net.Listener based on the config. func getListener(uri pnet.URI, tlsconf *tls.Config) (ln net.Listener, err error) { // If bind URI has the https scheme, enable TLS diff --git a/sql3/interfaces.go b/sql3/interfaces.go index 399449e8e..a9fd3bd1f 100644 --- a/sql3/interfaces.go +++ b/sql3/interfaces.go @@ -3,6 +3,7 @@ package sql3 import ( "context" + "io" "github.com/molecula/featurebase/v3/sql3/parser" "github.com/molecula/featurebase/v3/sql3/planner/types" @@ -10,6 +11,7 @@ import ( type CompilePlanner interface { CompilePlan(context.Context, parser.Statement) (types.PlanOperator, error) + RehydratePlanOp(context.Context, io.Reader) (types.PlanOperator, error) } // Ensure type implements interface. @@ -25,3 +27,7 @@ func NewNopCompilePlanner() *NopCompilePlanner { func (p *NopCompilePlanner) CompilePlan(ctx context.Context, stmt parser.Statement) (types.PlanOperator, error) { return nil, nil } + +func (p *NopCompilePlanner) RehydratePlanOp(ctx context.Context, reader io.Reader) (types.PlanOperator, error) { + return nil, nil +} diff --git a/sql3/planner/compilecreatetable.go b/sql3/planner/compilecreatetable.go index b2f211d27..4d9265998 100644 --- a/sql3/planner/compilecreatetable.go +++ b/sql3/planner/compilecreatetable.go @@ -196,7 +196,13 @@ func (p *ExecutionPlanner) compileColumn(col *parser.ColumnDefinition) (*createT case dax.BaseTypeBool: column.fos = append(column.fos, pilosa.OptFieldTypeBool()) case dax.BaseTypeDecimal: - // Get the scale value. + + // if we don't have a scale, it's an error + if col.Type.Scale == nil { + return nil, sql3.NewErrDecimalScaleExpected(col.Type.Name.NamePos.Line, col.Type.Name.NamePos.Column) + } + + // get the scale value scale, err = strconv.ParseInt(col.Type.Scale.Value, 10, 64) if err != nil { return nil, err diff --git a/sql3/planner/compileselect.go b/sql3/planner/compileselect.go index 25e26a911..2c58dadc2 100644 --- a/sql3/planner/compileselect.go +++ b/sql3/planner/compileselect.go @@ -302,11 +302,16 @@ func (p *ExecutionPlanner) compileSource(scope *PlanOpQuery, source parser.Sourc // doing this check here because we don't have a 'system' flag that exists in the FB schema st, ok := systemTables[strings.ToLower(tableName)] if ok { + var op types.PlanOperator + op = NewPlanOpSystemTable(p, st) + if st.requiresFanout { + op = NewPlanOpFanout(p, op) + } if sourceExpr.Alias != nil { aliasName := parser.IdentName(sourceExpr.Alias) - return NewPlanOpRelAlias(aliasName, NewPlanOpSystemTable(p, st)), nil + return NewPlanOpRelAlias(aliasName, op), nil } - return NewPlanOpSystemTable(p, st), nil + return op, nil } // get all the columns for this table - we will eliminate unused ones diff --git a/sql3/planner/executionplanner.go b/sql3/planner/executionplanner.go index 58579dff0..ac8fc56ee 100644 --- a/sql3/planner/executionplanner.go +++ b/sql3/planner/executionplanner.go @@ -3,13 +3,20 @@ package planner import ( + "bytes" "context" + "fmt" + "io" + "net/http" + "strconv" pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/logger" "github.com/molecula/featurebase/v3/sql3" "github.com/molecula/featurebase/v3/sql3/parser" "github.com/molecula/featurebase/v3/sql3/planner/types" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" ) // ExecutionPlanner compiles SQL text into a query plan @@ -79,6 +86,20 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen return rootOperator, err } +func (p *ExecutionPlanner) RehydratePlanOp(ctx context.Context, reader io.Reader) (types.PlanOperator, error) { + rdr := newWireProtocolParser(p, reader) + message, err := rdr.nextMessage() + if err != nil { + return nil, err + } + switch m := message.(type) { + case *messagePlanOp: + return m.op, nil + default: + return nil, sql3.NewErrInternalf("unexpected message type '%T'", message) + } +} + func (p *ExecutionPlanner) analyzePlan(stmt parser.Statement) error { switch stmt := stmt.(type) { case *parser.SelectStatement: @@ -120,3 +141,216 @@ const ( func (p *ExecutionPlanner) checkAccess(ctx context.Context, objectName string, _ accessType) error { return nil } + +type reduceFunc func(ctx context.Context, prev, v types.Rows) (types.Rows, error) + +type mapResponse struct { + node pilosa.ClusterNode + result types.Rows + err error +} + +func (e *ExecutionPlanner) mapReducePlanOp(ctx context.Context, op types.PlanOperator, reduceFn reduceFunc) (result types.Rows, err error) { + ch := make(chan mapResponse) + + // Wrap context with a cancel to kill goroutines on exit. + ctx, cancel := context.WithCancel(ctx) + // Create an errgroup so we can wait for all the goroutines to exit + eg, ctx := errgroup.WithContext(ctx) + + // After we're done processing, we have to wait for any outstanding + // functions in the ErrGroup to complete. If we didn't have an error + // already at that point, we'll report any errors from the ErrGroup + // instead. + defer func() { + cancel() + errWait := eg.Wait() + if err == nil { + err = errWait + } + }() + + nodes := e.systemAPI.ClusterNodes() + + // Start mapping across all nodes + if err = e.mapper(ctx, eg, ch, nodes, op, reduceFn); err != nil { + return nil, errors.Wrap(err, "starting mapper") + } + + // Iterate over all map responses and reduce. + expected := len(nodes) + done := ctx.Done() + for expected > 0 { + select { + case <-done: + return nil, ctx.Err() + case resp := <-ch: + if resp.err != nil { + return nil, errors.Wrap(resp.err, "query fanout") + } + // if we got a response that we aren't discarding + // because it's an error, subtract it from our count... + expected -= 1 + + // Reduce value. + + result, err = reduceFn(ctx, result, resp.result) + if err != nil { + cancel() + return nil, err + } + } + } + // note the deferred Wait above which might override this nil. + return result, nil +} + +func (e *ExecutionPlanner) mapper(ctx context.Context, eg *errgroup.Group, ch chan mapResponse, nodes []pilosa.ClusterNode, op types.PlanOperator, reduceFn reduceFunc) (reterr error) { + done := ctx.Done() + // Execute each node in a separate goroutine. + for _, node := range nodes { + node := node + eg.Go(func() error { + + resp := mapResponse{node: node} + + // Send local shards to mapper, otherwise remote exec. + if node.ID == e.systemAPI.NodeID() { + iter, err := op.Iterator(ctx, nil) + if err != nil { + resp.result = nil + resp.err = err + } + row, err := iter.Next(ctx) + if err != nil && err != types.ErrNoMoreRows { + resp.result = nil + resp.err = err + } + if err != types.ErrNoMoreRows { + for { + resp.result = append(resp.result, row) + row, err = iter.Next(ctx) + if err != nil && err != types.ErrNoMoreRows { + resp.result = nil + resp.err = err + } + if err == types.ErrNoMoreRows { + break + } + } + } + } else { + results, err := e.remotePlanExec(ctx, node.URI, op) + resp.result = results + resp.err = err + } + + // Return response to the channel. + select { + case <-done: + // If someone just canceled the context + // arbitrarily, we could end up here with this + // being the first non-nil error handed to + // the ErrGroup, in which case, it's the best + // explanation we have for why everything's + // stopping. + return ctx.Err() + case ch <- resp: + // If we return a non-nil error from this, the + // entire errGroup gets canceled. So we don't + // want to return a non-nil error if mapReduce + // might try to run another mapper against a + // different set of nodes. Note that this shouldn't + // matter; we just sent the error to mapReduce + // anyway, so it probably cancels the ErrGroup + // too. + if resp.err != nil { + return resp.err + } + } + return nil + }) + if reterr != nil { + return reterr // exit early if error occurs when running serially + } + } + return nil +} + +func (e *ExecutionPlanner) remotePlanExec(ctx context.Context, addr string, op types.PlanOperator) (types.Rows, error) { + b, err := writeOp(op) + if err != nil { + return nil, err + } + + // Create HTTP request. + u := fmt.Sprintf("%s/sql", addr) + req, err := http.NewRequest("POST", u, bytes.NewReader(b)) + if err != nil { + return nil, errors.Wrap(err, "creating request") + } + + // TODO (pok) internal auth + //AddAuthToken(ctx, &req.Header) + + req.Header.Set("Content-Length", strconv.Itoa(len(b))) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Accept", "application/octet-stream") + req.Header.Set("User-Agent", "pilosa/"+e.systemAPI.Version()) + req.Header.Set("X-FeatureBase-Plan-Operator", fmt.Sprintf("%T", op)) + + // Execute request against the host. + resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + return nil, sql3.NewErrInternalf("error posting internally: %s", err.Error()) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + // we have an error + return nil, sql3.NewErrInternalf("error posting internally: %d", resp.StatusCode) + } + + var rows types.Rows + parser := newWireProtocolParser(e, resp.Body) + state := 1 + for state <= 2 { + msg, err := parser.nextMessage() + if err != nil { + return nil, err + } + switch state { + case 1: + switch m := msg.(type) { + case *messageSchemaInfo: + parser.schema = m.schema + state = 2 + + case *messageError: + return nil, m.err + + default: + return nil, sql3.NewErrInternalf("unexpected token %d", msg.Token()) + } + + case 2: + switch m := msg.(type) { + case *messageRow: + rows = append(rows, m.row) + + case *messageDone: + // we're done + state = 3 + + case *messageError: + return nil, m.err + + default: + return nil, sql3.NewErrInternalf("unexpected token %d", msg.Token()) + } + + } + + } + return rows, nil +} diff --git a/sql3/planner/opbulkinsert.go b/sql3/planner/opbulkinsert.go index fecf4cddc..0e0c335b0 100644 --- a/sql3/planner/opbulkinsert.go +++ b/sql3/planner/opbulkinsert.go @@ -17,6 +17,7 @@ import ( "github.com/PaesslerAG/gval" "github.com/PaesslerAG/jsonpath" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/sql3" "github.com/molecula/featurebase/v3/sql3/parser" @@ -367,6 +368,8 @@ func (i *bulkInsertCSVRowIter) Next(ctx context.Context) (types.Row, error) { return nil, err } i.currentBatch = nil + // update the counter for bulk insert batches + pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1) } if i.options.rowsLimit > 0 && i.linesRead >= i.options.rowsLimit { break @@ -378,6 +381,8 @@ func (i *bulkInsertCSVRowIter) Next(ctx context.Context) (types.Row, error) { return nil, err } i.currentBatch = nil + // update the counter for bulk insert batches + pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1) } return nil, types.ErrNoMoreRows } @@ -777,6 +782,8 @@ func (i *bulkInsertNDJsonRowIter) Next(ctx context.Context) (types.Row, error) { return nil, err } i.currentBatch = nil + // update the counter for bulk insert batches + pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1) } if i.options.rowsLimit > 0 && i.linesRead >= i.options.rowsLimit { break @@ -788,6 +795,8 @@ func (i *bulkInsertNDJsonRowIter) Next(ctx context.Context) (types.Row, error) { return nil, err } i.currentBatch = nil + // update the counter for bulk insert batches + pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1) } return nil, types.ErrNoMoreRows } @@ -928,5 +937,9 @@ func processBatch(ctx context.Context, planner *ExecutionPlanner, tableName stri if err != nil && err != types.ErrNoMoreRows { return err } + + // update the counter for bulk inserts + pilosa.PerfCounterSQLBulkInsertsSec.Add(int64(len(insertValues))) + return nil } diff --git a/sql3/planner/opfanout.go b/sql3/planner/opfanout.go new file mode 100644 index 000000000..414b87418 --- /dev/null +++ b/sql3/planner/opfanout.go @@ -0,0 +1,114 @@ +// Copyright 2022 Molecula Corp. All rights reserved. + +package planner + +import ( + "context" + "fmt" + + "github.com/molecula/featurebase/v3/sql3" + "github.com/molecula/featurebase/v3/sql3/planner/types" +) + +// PlanOpFanout is a query fanout operator that will execute an operator across all cluster nodes +type PlanOpFanout struct { + planner *ExecutionPlanner + ChildOp types.PlanOperator + warnings []string +} + +func NewPlanOpFanout(planner *ExecutionPlanner, child types.PlanOperator) *PlanOpFanout { + return &PlanOpFanout{ + planner: planner, + ChildOp: child, + warnings: make([]string, 0), + } +} + +func (p *PlanOpFanout) Schema() types.Schema { + return p.ChildOp.Schema() +} + +func (p *PlanOpFanout) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) { + return newFanOutIterator(p.planner, p.ChildOp), nil +} + +func (p *PlanOpFanout) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) { + if len(children) != 1 { + return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children)) + } + return NewPlanOpFanout(p.planner, children[0]), nil +} + +func (p *PlanOpFanout) Children() []types.PlanOperator { + return []types.PlanOperator{ + p.ChildOp, + } +} + +func (p *PlanOpFanout) Plan() map[string]interface{} { + result := make(map[string]interface{}) + result["_op"] = fmt.Sprintf("%T", p) + ps := make([]string, 0) + for _, e := range p.Schema() { + ps = append(ps, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeDescription())) + } + result["_schema"] = ps + result["child"] = p.ChildOp.Plan() + return result +} + +func (p *PlanOpFanout) String() string { + return "" +} + +func (p *PlanOpFanout) AddWarning(warning string) { + p.warnings = append(p.warnings, warning) +} + +func (p *PlanOpFanout) Warnings() []string { + return p.warnings +} + +func (p *PlanOpFanout) Expressions() []types.PlanExpression { + return []types.PlanExpression{} +} + +func (p *PlanOpFanout) WithUpdatedExpressions(exprs ...types.PlanExpression) (types.PlanOperator, error) { + if len(exprs) != 1 { + return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs)) + } + return NewPlanOpFilter(p.planner, exprs[0], p.ChildOp), nil +} + +type fanOutIterator struct { + planner *ExecutionPlanner + childOp types.PlanOperator + rows types.Rows +} + +func newFanOutIterator(planner *ExecutionPlanner, childOp types.PlanOperator) *fanOutIterator { + return &fanOutIterator{ + planner: planner, + childOp: childOp, + } +} + +func (i *fanOutIterator) Next(ctx context.Context) (types.Row, error) { + if i.rows == nil { + rows, err := i.planner.mapReducePlanOp(ctx, i.childOp, func(ctx context.Context, prev, v types.Rows) (types.Rows, error) { + return append(prev, v...), nil + }) + if err != nil { + return nil, err + } + i.rows = rows + } + if len(i.rows) > 0 { + row := i.rows[0] + // Move to next result element. + i.rows = i.rows[1:] + return row, nil + } + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/opinsert.go b/sql3/planner/opinsert.go index 99dfa4ee7..b1211b201 100644 --- a/sql3/planner/opinsert.go +++ b/sql3/planner/opinsert.go @@ -368,5 +368,8 @@ func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) { return nil, errors.Wrap(err, "importing batch") } + // update the counter for inserts + pilosa.PerfCounterSQLInsertsSec.Add(int64(batch.Len())) + return nil, types.ErrNoMoreRows } diff --git a/sql3/planner/opsystemtable.go b/sql3/planner/opsystemtable.go index 51a7ca841..ba59e34e1 100644 --- a/sql3/planner/opsystemtable.go +++ b/sql3/planner/opsystemtable.go @@ -17,16 +17,18 @@ import ( // exclude this file from SonarCloud dupe eval const ( - fbClusterInfo = "fb_cluster_info" - fbClusterNodes = "fb_cluster_nodes" - fbExecRequests = "fb_exec_requests" + fbClusterInfo = "fb_cluster_info" + fbClusterNodes = "fb_cluster_nodes" + fbExecRequests = "fb_exec_requests" + fbPerformanceCounters = "fb_performance_counters" fbTableDDL = "fb_table_ddl" ) type systemTable struct { - name string - schema types.Schema + name string + schema types.Schema + requiresFanout bool } var systemTables = map[string]*systemTable{ @@ -74,6 +76,7 @@ var systemTables = map[string]*systemTable{ Type: parser.NewDataTypeInt(), }, }, + requiresFanout: false, }, fbClusterNodes: { name: fbClusterNodes, @@ -109,11 +112,17 @@ var systemTables = map[string]*systemTable{ Type: parser.NewDataTypeBool(), }, }, + requiresFanout: false, }, fbExecRequests: { name: fbExecRequests, schema: types.Schema{ + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "nodeid", + Type: parser.NewDataTypeString(), + }, &types.PlannerColumn{ RelationName: fbExecRequests, ColumnName: "request_id", @@ -195,6 +204,7 @@ var systemTables = map[string]*systemTable{ Type: parser.NewDataTypeString(), }, }, + requiresFanout: true, }, fbTableDDL: { @@ -216,6 +226,44 @@ var systemTables = map[string]*systemTable{ Type: parser.NewDataTypeString(), }, }, + requiresFanout: false, + }, + + fbPerformanceCounters: { + name: fbPerformanceCounters, + schema: types.Schema{ + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "nodeid", + Type: parser.NewDataTypeString(), + }, + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "namespace", + Type: parser.NewDataTypeString(), + }, + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "subsystem", + Type: parser.NewDataTypeString(), + }, + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "counter_name", + Type: parser.NewDataTypeString(), + }, + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "value", + Type: parser.NewDataTypeInt(), + }, + &types.PlannerColumn{ + RelationName: fbPerformanceCounters, + ColumnName: "counter_type", + Type: parser.NewDataTypeInt(), + }, + }, + requiresFanout: true, }, } @@ -279,6 +327,10 @@ func (p *PlanOpSystemTable) Iterator(ctx context.Context, row types.Row) (types. return &fbTableDDLRowIter{ planner: p.planner, }, nil + case fbPerformanceCounters: + return &fbPerformanceCountersRowIter{ + planner: p.planner, + }, nil default: return nil, sql3.NewErrInternalf("unable to find system table '%s'", p.table.name) } @@ -367,9 +419,11 @@ func (i *fbExecRequestsRowIter) Next(ctx context.Context) (types.Row, error) { } } + nodeId := i.planner.systemAPI.NodeID() if len(i.result) > 0 { n := i.result[0] row := []interface{}{ + nodeId, n.RequestID, n.UserID, n.StartTime, @@ -533,3 +587,37 @@ func (i *fbTableDDLRowIter) Next(ctx context.Context) (types.Row, error) { } return nil, types.ErrNoMoreRows } + +type fbPerformanceCountersRowIter struct { + planner *ExecutionPlanner + result []pilosa.PerformanceCounter +} + +var _ types.RowIterator = (*fbPerformanceCountersRowIter)(nil) + +func (i *fbPerformanceCountersRowIter) Next(ctx context.Context) (types.Row, error) { + if i.result == nil { + var err error + i.result, err = pilosa.PerfCounters.ListCounters() + if err != nil { + return nil, err + } + } + + nodeId := i.planner.systemAPI.NodeID() + if len(i.result) > 0 { + n := i.result[0] + row := []interface{}{ + nodeId, + n.NameSpace, + n.SubSystem, + n.CounterName, + n.Value, + n.CounterType, + } + // Move to next result element. + i.result = i.result[1:] + return row, nil + } + return nil, types.ErrNoMoreRows +} diff --git a/sql3/planner/types/operator.go b/sql3/planner/types/operator.go index df38de132..1e9d4faab 100644 --- a/sql3/planner/types/operator.go +++ b/sql3/planner/types/operator.go @@ -80,9 +80,12 @@ func (r Schema) Plan() []map[string]interface{} { return result } -// Row is a tuple of values +// Row is a tuple (of values) type Row []interface{} +// Rows is a table of rows +type Rows []Row + // Append appends all the values in r2 to this row and returns the result func (r Row) Append(r2 Row) Row { row := make(Row, len(r)+len(r2)) diff --git a/sql3/planner/wireprotocol.go b/sql3/planner/wireprotocol.go new file mode 100644 index 000000000..a742a5e91 --- /dev/null +++ b/sql3/planner/wireprotocol.go @@ -0,0 +1,211 @@ +package planner + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "io" + + "github.com/molecula/featurebase/v3/sql3" + "github.com/molecula/featurebase/v3/sql3/planner/types" + "github.com/molecula/featurebase/v3/wireprotocol" +) + +type wireProtocolMessage interface { + Token() int16 +} + +func writeOp(op types.PlanOperator) ([]byte, error) { + buf := new(bytes.Buffer) + writer := bufio.NewWriter(buf) + // serialize a plan op - for now we are just supporting + // system tables, and we'll send the name of the system table + switch op := op.(type) { + case *PlanOpSystemTable: + // write token + b := make([]byte, 2) + binary.BigEndian.PutUint16(b, uint16(wireprotocol.TOKEN_PLAN_OP)) + writer.Write(b) + // write table name len + t := op.table.name + b = make([]byte, 4) + binary.BigEndian.PutUint32(b, uint32(len(t))) + writer.Write(b) + // write table name + writer.WriteString(op.table.name) + writer.Flush() + default: + return []byte{}, sql3.NewErrInternalf("unexpected plan operator type '%T'", op) + } + return buf.Bytes(), nil +} + +type wireProtocolParser struct { + planner *ExecutionPlanner + reader io.Reader + schema types.Schema +} + +func newWireProtocolParser(p *ExecutionPlanner, reader io.Reader) *wireProtocolParser { + return &wireProtocolParser{ + planner: p, + reader: reader, + } +} + +func (f *wireProtocolParser) nextMessage() (wireProtocolMessage, error) { + var t int16 + err := binary.Read(f.reader, binary.BigEndian, &t) + if err != nil { + return nil, err + } + switch t { + case wireprotocol.TOKEN_SCHEMA_INFO: + return newMessageSchemaInfo(f.planner, f.reader) + + case wireprotocol.TOKEN_ROW: + if f.schema == nil { + return nil, sql3.NewErrInternalf("schema uninitialized") + } + return newMessageRow(f.planner, f.reader, f.schema) + + case wireprotocol.TOKEN_ERROR_MESSAGE: + return newMessageError(f.planner, f.reader) + + case wireprotocol.TOKEN_DONE: + return newMessageDone(f.planner, f.reader) + + case wireprotocol.TOKEN_PLAN_OP: + return newMessagePlanOp(f.planner, f.reader) + + default: + return nil, sql3.NewErrInternalf("unexpected token %d", t) + } +} + +type messagePlanOp struct { + token int16 + op types.PlanOperator +} + +var _ wireProtocolMessage = (*messagePlanOp)(nil) + +func newMessagePlanOp(p *ExecutionPlanner, reader io.Reader) (*messagePlanOp, error) { + + var len int32 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + bname := make([]byte, len) + err = binary.Read(reader, binary.BigEndian, &bname) + if err != nil { + return nil, err + } + name := string(bname) + + st, ok := systemTables[name] + if !ok { + return nil, sql3.NewErrInternalf("unexpected system table name %s", name) + } + + return &messagePlanOp{ + token: wireprotocol.TOKEN_PLAN_OP, + op: NewPlanOpSystemTable(p, st), + }, nil +} + +func (m *messagePlanOp) Token() int16 { + return m.token +} + +type messageError struct { + token int16 + err error +} + +var _ wireProtocolMessage = (*messageError)(nil) + +func newMessageError(p *ExecutionPlanner, reader io.Reader) (*messageError, error) { + var len int32 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + bname := make([]byte, len) + err = binary.Read(reader, binary.BigEndian, &bname) + if err != nil { + return nil, err + } + errMsg := string(bname) + + return &messageError{ + token: wireprotocol.TOKEN_ERROR_MESSAGE, + err: errors.New(errMsg), + }, nil +} + +func (m *messageError) Token() int16 { + return m.token +} + +type messageSchemaInfo struct { + token int16 + schema types.Schema +} + +var _ wireProtocolMessage = (*messageSchemaInfo)(nil) + +func newMessageSchemaInfo(p *ExecutionPlanner, reader io.Reader) (*messageSchemaInfo, error) { + schema, err := wireprotocol.ReadSchema(reader) + if err != nil { + return nil, err + } + return &messageSchemaInfo{ + token: wireprotocol.TOKEN_SCHEMA_INFO, + schema: schema, + }, nil +} + +func (m *messageSchemaInfo) Token() int16 { + return m.token +} + +type messageRow struct { + token int16 + row types.Row +} + +var _ wireProtocolMessage = (*messageRow)(nil) + +func newMessageRow(p *ExecutionPlanner, reader io.Reader, schema types.Schema) (*messageRow, error) { + row, err := wireprotocol.ReadRow(reader, schema) + if err != nil { + return nil, err + } + return &messageRow{ + token: wireprotocol.TOKEN_ROW, + row: row, + }, nil +} + +func (m *messageRow) Token() int16 { + return m.token +} + +type messageDone struct { + token int16 +} + +var _ wireProtocolMessage = (*messageDone)(nil) + +func newMessageDone(p *ExecutionPlanner, reader io.Reader) (*messageDone, error) { + return &messageDone{ + token: wireprotocol.TOKEN_DONE, + }, nil +} + +func (m *messageDone) Token() int16 { + return m.token +} diff --git a/sql3/sql_complex_test.go b/sql3/sql_complex_test.go index b33e4aeec..5dfb392d0 100644 --- a/sql3/sql_complex_test.go +++ b/sql3/sql_complex_test.go @@ -36,6 +36,92 @@ func TestPlanner_Misc(t *testing.T) { assert.True(t, d.EqualTo(pql.NewDecimal(12345678, 6))) } +func TestPlanner_SystemTableFanout(t *testing.T) { + c := test.MustRunCluster(t, 3) + defer c.Close() + + server := c.GetNode(0).Server + + t.Run("PerfCounters", func(t *testing.T) { + results, columns, err := sql_test.MustQueryRows(t, server, `select * from fb_performance_counters`) + if err != nil { + t.Fatal(err) + } + if len(results) != 15 { + t.Fatal(fmt.Errorf("unexpected result set length")) + } + + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldString("nodeid"), + wireQueryFieldString("namespace"), + wireQueryFieldString("subsystem"), + wireQueryFieldString("counter_name"), + wireQueryFieldInt("value"), + wireQueryFieldInt("counter_type"), + }, columns); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("SystemTablesExecRequests", func(t *testing.T) { + results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select * from fb_exec_requests`) + if err != nil { + t.Fatal(err) + } + + if len(results) != 2 { + t.Fatal(fmt.Errorf("unexpected result set length")) + } + + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldString("nodeid"), + wireQueryFieldString("request_id"), + wireQueryFieldString("user"), + wireQueryFieldTimestamp("start_time"), + wireQueryFieldTimestamp("end_time"), + wireQueryFieldString("status"), + wireQueryFieldString("wait_type"), + wireQueryFieldInt("wait_time"), + wireQueryFieldString("wait_resource"), + wireQueryFieldInt("cpu_time"), + wireQueryFieldInt("elapsed_time"), + wireQueryFieldInt("reads"), + wireQueryFieldInt("writes"), + wireQueryFieldInt("logical_reads"), + wireQueryFieldInt("row_count"), + wireQueryFieldString("sql"), + wireQueryFieldString("plan"), + }, columns); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("SystemTablesExecRequestsAgg", func(t *testing.T) { + _, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select + count(request_id) as request_count, + min(elapsed_time) as min_duration, + max(elapsed_time) as max_duration, + avg(elapsed_time) as avg_duration + from + fb_exec_requests + where + status = 'complete';`) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff([]*pilosa.WireQueryField{ + wireQueryFieldInt("request_count"), + wireQueryFieldInt("min_duration"), + wireQueryFieldInt("max_duration"), + wireQueryFieldDecimal("avg_duration", 4), + }, columns); diff != "" { + t.Fatal(diff) + } + }) + +} + func TestPlanner_Show(t *testing.T) { c := test.MustRunCluster(t, 1) defer c.Close() @@ -102,64 +188,12 @@ func TestPlanner_Show(t *testing.T) { } }) - t.Run("SystemTablesExecRequests", func(t *testing.T) { - _, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select * from fb_exec_requests`) - if err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff([]*pilosa.WireQueryField{ - wireQueryFieldString("request_id"), - wireQueryFieldString("user"), - wireQueryFieldTimestamp("start_time"), - wireQueryFieldTimestamp("end_time"), - wireQueryFieldString("status"), - wireQueryFieldString("wait_type"), - wireQueryFieldInt("wait_time"), - wireQueryFieldString("wait_resource"), - wireQueryFieldInt("cpu_time"), - wireQueryFieldInt("elapsed_time"), - wireQueryFieldInt("reads"), - wireQueryFieldInt("writes"), - wireQueryFieldInt("logical_reads"), - wireQueryFieldInt("row_count"), - wireQueryFieldString("sql"), - wireQueryFieldString("plan"), - }, columns); diff != "" { - t.Fatal(diff) - } - }) - - t.Run("SystemTablesExecRequestsAgg", func(t *testing.T) { - _, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select - count(request_id) as request_count, - min(elapsed_time) as min_duration, - max(elapsed_time) as max_duration, - avg(elapsed_time) as avg_duration - from - fb_exec_requests - where - status = 'complete';`) - if err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff([]*pilosa.WireQueryField{ - wireQueryFieldInt("request_count"), - wireQueryFieldInt("min_duration"), - wireQueryFieldInt("max_duration"), - wireQueryFieldDecimal("avg_duration", 4), - }, columns); diff != "" { - t.Fatal(diff) - } - }) - t.Run("ShowTables", func(t *testing.T) { results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SHOW TABLES`) if err != nil { t.Fatal(err) } - if len(results) != 6 { + if len(results) != 7 { t.Fatal(fmt.Errorf("unexpected result set length")) } diff --git a/stats/stats.go b/stats/stats.go deleted file mode 100644 index ba634de72..000000000 --- a/stats/stats.go +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package stats - -import ( - "expvar" - "sort" - "strings" - "sync" - "time" - - "github.com/molecula/featurebase/v3/logger" -) - -// Expvar global expvar map. -var Expvar *expvar.Map - -// StatsClient represents a client to a stats server. -type StatsClient interface { - // Returns a sorted list of tags on the client. - Tags() []string - - // Returns a new client with additional tags appended. - WithTags(tags ...string) StatsClient - - // Tracks the number of times something occurs per second. - Count(name string, value int64, rate float64) - - // Tracks the number of times something occurs per second with custom tags - CountWithCustomTags(name string, value int64, rate float64, tags []string) - - // Sets the value of a metric. - Gauge(name string, value float64, rate float64) - - // Tracks statistical distribution of a metric. - Histogram(name string, value float64, rate float64) - - // Tracks number of unique elements. - Set(name string, value string, rate float64) - - // Tracks timing information for a metric. - Timing(name string, value time.Duration, rate float64) - - // SetLogger Set the logger output type - SetLogger(logger logger.Logger) - - // Starts the service - Open() - - // Closes the client - Close() error -} - -// NopStatsClient represents a client that doesn't do anything. -var NopStatsClient StatsClient = &nopStatsClient{} - -type nopStatsClient struct{} - -func (c *nopStatsClient) Tags() []string { return nil } -func (c *nopStatsClient) WithTags(tags ...string) StatsClient { return c } -func (c *nopStatsClient) Count(name string, value int64, rate float64) {} -func (c *nopStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) {} -func (c *nopStatsClient) Gauge(name string, value float64, rate float64) {} -func (c *nopStatsClient) Histogram(name string, value float64, rate float64) {} -func (c *nopStatsClient) Set(name string, value string, rate float64) {} -func (c *nopStatsClient) Timing(name string, value time.Duration, rate float64) {} -func (c *nopStatsClient) SetLogger(logger logger.Logger) {} -func (c *nopStatsClient) Open() {} -func (c *nopStatsClient) Close() error { return nil } - -// expvarStatsClient writes stats out to expvars. -type expvarStatsClient struct { - mu sync.Mutex - m *expvar.Map - tags []string -} - -// NewExpvarStatsClient returns a new instance of ExpvarStatsClient. -// This client points at the root of the expvar index map. -func NewExpvarStatsClient() *expvarStatsClient { - if Expvar == nil { - Expvar = expvar.NewMap("index") - } - return &expvarStatsClient{ - m: Expvar, - } -} - -// Tags returns a sorted list of tags on the client. -func (c *expvarStatsClient) Tags() []string { - return nil -} - -// WithTags returns a new client with additional tags appended. -func (c *expvarStatsClient) WithTags(tags ...string) StatsClient { - m := &expvar.Map{} - m.Init() - c.m.Set(strings.Join(tags, ","), m) - - return &expvarStatsClient{ - m: m, - tags: unionStringSlice(c.tags, tags), - } -} - -// Count tracks the number of times something occurs. -func (c *expvarStatsClient) Count(name string, value int64, rate float64) { - c.m.Add(name, value) -} - -// CountWithCustomTags Tracks the number of times something occurs per second with custom tags -func (c *expvarStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) { - c.m.Add(name, value) -} - -// Gauge sets the value of a metric. -func (c *expvarStatsClient) Gauge(name string, value float64, rate float64) { - var f expvar.Float - f.Set(value) - c.m.Set(name, &f) -} - -// Histogram tracks statistical distribution of a metric. -// This works the same as gauge for this client. -func (c *expvarStatsClient) Histogram(name string, value float64, rate float64) { - c.Gauge(name, value, rate) -} - -// Set tracks number of unique elements. -func (c *expvarStatsClient) Set(name string, value string, rate float64) { - var s expvar.String - s.Set(value) - c.m.Set(name, &s) -} - -// Timing tracks timing information for a metric. -func (c *expvarStatsClient) Timing(name string, value time.Duration, rate float64) { - c.mu.Lock() - d, _ := c.m.Get(name).(time.Duration) - c.m.Set(name, d+value) - c.mu.Unlock() -} - -// SetLogger has no logger. -func (c *expvarStatsClient) SetLogger(logger logger.Logger) { -} - -// Open no-op. -func (c *expvarStatsClient) Open() {} - -// Close no-op. -func (c *expvarStatsClient) Close() error { return nil } - -// MultiStatsClient joins multiple stats clients together. -type MultiStatsClient []StatsClient - -// Tags returns tags from the first client. -func (a MultiStatsClient) Tags() []string { - if len(a) > 0 { - return a[0].Tags() - } - return nil -} - -// WithTags returns a new set of clients with the additional tags. -func (a MultiStatsClient) WithTags(tags ...string) StatsClient { - other := make(MultiStatsClient, len(a)) - for i := range a { - other[i] = a[i].WithTags(tags...) - } - return other -} - -// Count tracks the number of times something occurs per second on all clients. -func (a MultiStatsClient) Count(name string, value int64, rate float64) { - for _, c := range a { - c.Count(name, value, rate) - } -} - -// CountWithCustomTags Tracks the number of times something occurs per second with custom tags -func (a MultiStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) { - for _, c := range a { - c.CountWithCustomTags(name, value, rate, tags) - } -} - -// Gauge sets the value of a metric on all clients. -func (a MultiStatsClient) Gauge(name string, value float64, rate float64) { - for _, c := range a { - c.Gauge(name, value, rate) - } -} - -// Histogram tracks statistical distribution of a metric on all clients. -func (a MultiStatsClient) Histogram(name string, value float64, rate float64) { - for _, c := range a { - c.Histogram(name, value, rate) - } -} - -// Set tracks number of unique elements on all clients. -func (a MultiStatsClient) Set(name string, value string, rate float64) { - for _, c := range a { - c.Set(name, value, rate) - } -} - -// Timing tracks timing information for a metric on all clients. -func (a MultiStatsClient) Timing(name string, value time.Duration, rate float64) { - for _, c := range a { - c.Timing(name, value, rate) - } -} - -// SetLogger Sets the StatsD logger output type. -func (a MultiStatsClient) SetLogger(logger logger.Logger) { - for _, c := range a { - c.SetLogger(logger) - } -} - -// Open starts the stat service. -func (a MultiStatsClient) Open() { - for _, c := range a { - c.Open() - } -} - -// Close shuts down the stats clients. -func (a MultiStatsClient) Close() error { - for _, c := range a { - err := c.Close() - if err != nil { - return err - } - } - return nil -} - -// unionStringSlice returns a sorted set of tags which combine a & b. -func unionStringSlice(a, b []string) []string { - // Sort both sets first. - sort.Strings(a) - sort.Strings(b) - - // Find size of largest slice. - n := len(a) - if len(b) > n { - n = len(b) - } - - // Exit if both sets are empty. - if n == 0 { - return nil - } - - // Iterate over both in order and merge. - other := make([]string, 0, n) - for len(a) > 0 || len(b) > 0 { - if len(a) == 0 { - other, b = append(other, b[0]), b[1:] - } else if len(b) == 0 { - other, a = append(other, a[0]), a[1:] - } else if a[0] < b[0] { - other, a = append(other, a[0]), a[1:] - } else if b[0] < a[0] { - other, b = append(other, b[0]), b[1:] - } else { - other, a, b = append(other, a[0]), a[1:], b[1:] - } - } - return other -} diff --git a/stats/stats_test.go b/stats/stats_test.go deleted file mode 100644 index 8f4de8ab2..000000000 --- a/stats/stats_test.go +++ /dev/null @@ -1,257 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package stats_test - -import ( - "context" - "fmt" - "net/http/httptest" - "strings" - "testing" - "time" - - pilosa "github.com/molecula/featurebase/v3" - "github.com/molecula/featurebase/v3/logger" - "github.com/molecula/featurebase/v3/stats" - "github.com/molecula/featurebase/v3/test" -) - -// TestMultiStatClient_Expvar run the multistat client with exp var -// since the EXPVAR data is stored in a global we should run these in one test function -func TestMultiStatClient_Expvar(t *testing.T) { - hldr := test.MustOpenHolder(t) - - c := stats.NewExpvarStatsClient() - ms := make(stats.MultiStatsClient, 1) - ms[0] = c - hldr.Stats = ms - - hldr.SetBit("d", "f", 0, 0) - hldr.SetBit("d", "f", 0, 1) - hldr.SetBit("d", "f", 0, pilosa.ShardWidth) - hldr.SetBit("d", "f", 0, pilosa.ShardWidth+2) - hldr.ClearBit("d", "f", 0, 1) - - indexStats := fmt.Sprintf(`{"%s": %d, "%s": %d}`, pilosa.MetricClearBit, 1, pilosa.MetricSetBit, 4) - - if stats.Expvar.String() != `{"index:d": `+indexStats+`}` { - t.Fatalf("unexpected expvar : %s", stats.Expvar.String()) - } - - hldr.Stats.CountWithCustomTags("cc", 1, 1.0, []string{"foo:bar"}) - if stats.Expvar.String() != `{"cc": 1, "index:d": `+indexStats+`}` { - t.Fatalf("unexpected expvar : %s", stats.Expvar.String()) - } - - // Gauge creates a unique key, subsequent Gauge calls will overwrite - hldr.Stats.Gauge("g", 5, 1.0) - hldr.Stats.Gauge("g", 8, 1.0) - if stats.Expvar.String() != `{"cc": 1, "g": 8, "index:d": `+indexStats+`}` { - t.Fatalf("unexpected expvar : %s", stats.Expvar.String()) - } - - // Set creates a unique key, subsequent sets will overwrite - hldr.Stats.Set("s", "4", 1.0) - hldr.Stats.Set("s", "7", 1.0) - if stats.Expvar.String() != `{"cc": 1, "g": 8, "index:d": `+indexStats+`, "s": "7"}` { - t.Fatalf("unexpected expvar : %s", stats.Expvar.String()) - } - - // Record timing duration and a uniquely Set key/value - dur, _ := time.ParseDuration("123us") - hldr.Stats.Timing("tt", dur, 1.0) - if stats.Expvar.String() != `{"cc": 1, "g": 8, "index:d": `+indexStats+`, "s": "7", "tt": 123µs}` { - t.Fatalf("unexpected expvar : %s", stats.Expvar.String()) - } - - // Expvar histogram is implemented as a gauge - hldr.Stats.Histogram("hh", 3, 1.0) - if stats.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": `+indexStats+`, "s": "7", "tt": 123µs}` { - t.Fatalf("unexpected expvar : %s", stats.Expvar.String()) - } - - // Expvar should ignore earlier set tags from setbit - if hldr.Stats.Tags() != nil { - t.Fatalf("unexpected tag") - } -} - -func TestStatsCount_TopN(t *testing.T) { - c := test.MustRunCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()} - - // Execute query. - called := false - hldr.Holder.Stats = &MockStats{ - mockCountWithTags: func(name string, value int64, rate float64, tags []string) { - if name != "query_topn_total" { - t.Errorf("Expected query_topn_total, Results %s", name) - } - - if tags[0] != "index:d" { - t.Errorf("Expected index, Results %s", tags[0]) - } - - called = true - }, - } - - hldr.SetBit("d", "f", 0, 0) - hldr.SetBit("d", "f", 0, 1) - hldr.SetBit("d", "f", 0, pilosa.ShardWidth) - hldr.SetBit("d", "f", 0, pilosa.ShardWidth+2) - - if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `TopN(field=f, n=2)`}); err != nil { - t.Fatal(err) - } - if !called { - t.Error("CountWithCustomTags name isn't called") - } -} - -func TestStatsCount_Bitmap(t *testing.T) { - // Cluster has to be unhsared because we're mocking the stats which writes - // to a holder in use by other tests. - c := test.MustRunUnsharedCluster(t, 1) - defer c.Close() - hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()} - called := false - hldr.Holder.Stats = &MockStats{ - mockCountWithTags: func(name string, value int64, rate float64, tags []string) { - if name != pilosa.MetricRow { - t.Errorf("Expected %s, Results %s", pilosa.MetricRow, name) - } - - if tags[0] != "index:d" { - t.Errorf("Expected index, Results %s", tags[0]) - } - - called = true - }, - } - - hldr.SetBit("d", "f", 0, 0) - hldr.SetBit("d", "f", 0, 1) - - if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `Row(f=0)`}); err != nil { - t.Fatal(err) - } - if !called { - t.Error("CountWithCustomTags name isn't called") - } -} - -func TestStatsCount_APICalls(t *testing.T) { - // We can't share a cluster when we're modifying its stats counter. - cluster := test.MustRunUnsharedCluster(t, 1) - defer cluster.Close() - cmd := cluster.GetNode(0) - h := cmd.Handler.(*pilosa.Handler).Handler - holder := cmd.Server.Holder() - hldr := test.Holder{Holder: holder} - - t.Run("create index", func(t *testing.T) { - called := false - hldr.Stats = &MockStats{ - mockCount: func(name string, value int64, rate float64) { - if name != pilosa.MetricCreateIndex { - t.Errorf("Expected %v, Results %s", pilosa.MetricCreateIndex, name) - } - called = true - }, - } - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i", strings.NewReader(""))) - if !called { - t.Error("Count isn't called") - } - }) - - t.Run("create field", func(t *testing.T) { - called := false - hldr.Stats = &MockStats{ - mockCountWithTags: func(name string, value int64, rate float64, index []string) { - if name != pilosa.MetricCreateField { - t.Errorf("Expected %v, Results %s", pilosa.MetricCreateField, name) - } - if index[0] != "index:i" { - t.Errorf("Expected index:i, Results %s", index) - } - - called = true - }, - } - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/field/f", strings.NewReader(""))) - if !called { - t.Error("Count isn't called") - } - }) - - t.Run("delete field", func(t *testing.T) { - called := false - hldr.Stats = &MockStats{ - mockCountWithTags: func(name string, value int64, rate float64, index []string) { - if name != pilosa.MetricDeleteField { - t.Errorf("Expected %v, Results %s", pilosa.MetricDeleteField, name) - } - if index[0] != "index:i" { - t.Errorf("Expected index:i, Results %s", index) - } - - called = true - }, - } - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f", strings.NewReader(""))) - if !called { - t.Error("Count isn't called") - } - }) - - t.Run("delete index", func(t *testing.T) { - called := false - hldr.Stats = &MockStats{ - mockCount: func(name string, value int64, rate float64) { - if name != pilosa.MetricDeleteIndex { - t.Errorf("Expected %v, Results %s", pilosa.MetricDeleteIndex, name) - } - - called = true - }, - } - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader(""))) - if !called { - t.Error("Count isn't called") - } - }) - -} - -type MockStats struct { - mockCount func(name string, value int64, rate float64) - mockCountWithTags func(name string, value int64, rate float64, tags []string) -} - -func (s *MockStats) Count(name string, value int64, rate float64) { - if s.mockCount != nil { - s.mockCount(name, value, rate) - } -} - -func (s *MockStats) CountWithCustomTags(name string, value int64, rate float64, tags []string) { - if s.mockCountWithTags != nil { - s.mockCountWithTags(name, value, rate, tags) - } -} - -func (c *MockStats) Tags() []string { return nil } -func (c *MockStats) WithTags(tags ...string) stats.StatsClient { return c } -func (c *MockStats) Gauge(name string, value float64, rate float64) {} -func (c *MockStats) Histogram(name string, value float64, rate float64) {} -func (c *MockStats) Set(name string, value string, rate float64) {} -func (c *MockStats) Timing(name string, value time.Duration, rate float64) {} -func (c *MockStats) SetLogger(logger logger.Logger) {} -func (c *MockStats) Open() {} -func (c *MockStats) Close() error { return nil } diff --git a/statsd/statsd.go b/statsd/statsd.go deleted file mode 100644 index e9975f79f..000000000 --- a/statsd/statsd.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package statsd - -import ( - "sort" - "time" - - "github.com/DataDog/datadog-go/statsd" - "github.com/molecula/featurebase/v3/logger" - "github.com/molecula/featurebase/v3/stats" -) - -// StatsD protocol wrapper using the DataDog library that added Tags to the StatsD protocol -// statsD defailt host is "127.0.0.1:8125" - -const ( - // bufferLen Stats lient buffer size. - bufferLen = 1024 -) - -// Ensure client implements interface. -var _ stats.StatsClient = &statsClient{} - -// statsClient represents a StatsD implementation of pilosa.statsClient. -type statsClient struct { - client *statsd.Client - tags []string - logger logger.Logger - - // prefix is appended to each metric event name - prefix string -} - -// NewStatsClient returns a new instance of StatsClient. -func NewStatsClient(host string, namespace string) (*statsClient, error) { - c, err := statsd.NewBuffered(host, bufferLen) - if err != nil { - return nil, err - } - - return &statsClient{ - client: c, - logger: logger.NopLogger, - prefix: namespace + ".", - }, nil -} - -// Open no-op -func (c *statsClient) Open() {} - -// Close closes the connection to the agent. -func (c *statsClient) Close() error { - return c.client.Close() -} - -// Tags returns a sorted list of tags on the client. -func (c *statsClient) Tags() []string { - return c.tags -} - -// WithTags returns a new client with additional tags appended. -func (c *statsClient) WithTags(tags ...string) stats.StatsClient { - return &statsClient{ - client: c.client, - tags: unionStringSlice(c.tags, tags), - logger: c.logger, - } -} - -// Count tracks the number of times something occurs per second. -func (c *statsClient) Count(name string, value int64, rate float64) { - if err := c.client.Count(c.prefix+name, value, c.tags, rate); err != nil { - c.logger.Errorf("statsd.StatsClient.Count error: %s", err) - } -} - -// CountWithCustomTags tracks the number of times something occurs per second with custom tags. -func (c *statsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) { - tags := append(c.tags, t...) - if err := c.client.Count(c.prefix+name, value, tags, rate); err != nil { - c.logger.Errorf("statsd.StatsClient.Count error: %s", err) - } -} - -// Gauge sets the value of a metric. -func (c *statsClient) Gauge(name string, value float64, rate float64) { - if err := c.client.Gauge(c.prefix+name, value, c.tags, rate); err != nil { - c.logger.Errorf("statsd.StatsClient.Gauge error: %s", err) - } -} - -// Histogram tracks statistical distribution of a metric. -func (c *statsClient) Histogram(name string, value float64, rate float64) { - if err := c.client.Histogram(c.prefix+name, value, c.tags, rate); err != nil { - c.logger.Errorf("statsd.StatsClient.Histogram error: %s", err) - } -} - -// Set tracks number of unique elements. -func (c *statsClient) Set(name string, value string, rate float64) { - if err := c.client.Set(c.prefix+name, value, c.tags, rate); err != nil { - c.logger.Errorf("statsd.StatsClient.Set error: %s", err) - } -} - -// Timing tracks timing information for a metric. -func (c *statsClient) Timing(name string, value time.Duration, rate float64) { - if err := c.client.Timing(c.prefix+name, value, c.tags, rate); err != nil { - c.logger.Errorf("statsd.StatsClient.Timing error: %s", err) - } -} - -// SetLogger sets the logger for client. -func (c *statsClient) SetLogger(logger logger.Logger) { - c.logger = logger -} - -// unionStringSlice returns a sorted set of tags which combine a & b. -func unionStringSlice(a, b []string) []string { - // Sort both sets first. - sort.Strings(a) - sort.Strings(b) - - // Find size of largest slice. - n := len(a) - if len(b) > n { - n = len(b) - } - - // Exit if both sets are empty. - if n == 0 { - return nil - } - - // Iterate over both in order and merge. - other := make([]string, 0, n) - for len(a) > 0 || len(b) > 0 { - if len(a) == 0 { - other, b = append(other, b[0]), b[1:] - } else if len(b) == 0 { - other, a = append(other, a[0]), a[1:] - } else if a[0] < b[0] { - other, a = append(other, a[0]), a[1:] - } else if b[0] < a[0] { - other, b = append(other, b[0]), b[1:] - } else { - other, a, b = append(other, a[0]), a[1:], b[1:] - } - } - return other -} diff --git a/statsd/statsd_test.go b/statsd/statsd_test.go deleted file mode 100644 index 8c8b8e43a..000000000 --- a/statsd/statsd_test.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2021 Molecula Corp. All rights reserved. -package statsd_test - -import ( - "reflect" - "testing" - "time" - - "github.com/molecula/featurebase/v3/statsd" - _ "github.com/molecula/featurebase/v3/test" -) - -func TestStatsClient_WithTags(t *testing.T) { - // Create a new client. - c, err := statsd.NewStatsClient("localhost:19444", "testnamespace") - if err != nil { - t.Fatal(err) - } - defer c.Close() - - // Create a new client with additional tags. - c1 := c.WithTags("foo", "bar") - if tags := c1.Tags(); !reflect.DeepEqual(tags, []string{"bar", "foo"}) { - t.Fatalf("unexpected tags: %+v", tags) - } - - // Create a new client from the clone with more tags. - c2 := c1.WithTags("bar", "baz") - if tags := c2.Tags(); !reflect.DeepEqual(tags, []string{"bar", "baz", "foo"}) { - t.Fatalf("unexpected tags: %+v", tags) - } -} - -func TestStatsClient_Methods(t *testing.T) { - // Create a new client. - c, err := statsd.NewStatsClient("localhost:19444", "testnamespace") - if err != nil { - t.Fatal(err) - } - defer c.Close() - - dur, _ := time.ParseDuration("123us") - c.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"}) - c.Count("cc", 1, 1.0) - c.Gauge("gg", 10, 1.0) - c.Histogram("hh", 1, 1.0) - c.Timing("tt", dur, 1.0) - c.Set("ss", "ss", 1.0) -} diff --git a/view.go b/view.go index 9fc727cde..7a6fccd5e 100644 --- a/view.go +++ b/view.go @@ -15,7 +15,6 @@ import ( "github.com/molecula/featurebase/v3/pql" "github.com/molecula/featurebase/v3/roaring" - "github.com/molecula/featurebase/v3/stats" "github.com/molecula/featurebase/v3/testhook" "github.com/molecula/featurebase/v3/vprint" "github.com/pkg/errors" @@ -50,7 +49,6 @@ type view struct { fragments map[uint64]*fragment broadcaster broadcaster - stats stats.StatsClient knownShards *roaring.Bitmap knownShardsCopied uint32 @@ -78,7 +76,6 @@ func newView(holder *Holder, path, index, field, name string, fieldOptions Field fragments: make(map[uint64]*fragment), broadcaster: NopBroadcaster, - stats: stats.NopStatsClient, knownShards: roaring.NewSliceBitmap(), closing: make(chan struct{}), @@ -392,7 +389,6 @@ func (v *view) newFragment(shard uint64) *fragment { frag := newFragment(v.holder, v.idx, v.fld, v, shard) frag.CacheType = v.cacheType frag.CacheSize = v.cacheSize - frag.stats = v.stats if v.fieldType == FieldTypeMutex { frag.mutexVector = newRowsVector(frag) } else if v.fieldType == FieldTypeBool { diff --git a/wireprotocol/wireprimitives.go b/wireprotocol/wireprimitives.go new file mode 100644 index 000000000..228d523c1 --- /dev/null +++ b/wireprotocol/wireprimitives.go @@ -0,0 +1,563 @@ +package wireprotocol + +import ( + "bufio" + "bytes" + "encoding/binary" + "time" + + "io" + + "github.com/molecula/featurebase/v3/errors" + "github.com/molecula/featurebase/v3/pql" + "github.com/molecula/featurebase/v3/sql3/parser" + "github.com/molecula/featurebase/v3/sql3/planner/types" +) + +const ( + // server --> client + TOKEN_SCHEMA_INFO int16 = 0xA1 + TOKEN_ROW int16 = 0xA2 + TOKEN_DONE int16 = 0xFD + TOKEN_INFO_MESSAGE int16 = 0xFE + TOKEN_ERROR_MESSAGE int16 = 0xFF + + // client --> server + TOKEN_SQL int16 = 0x01 + TOKEN_PLAN_OP int16 = 0x02 +) + +const ( + TYPE_VOID int8 = 0x00 + TYPE_ID int8 = 0x01 + TYPE_BOOL int8 = 0x02 + TYPE_INT int8 = 0x03 + TYPE_DECIMAL int8 = 0x04 + TYPE_TIMESTAMP int8 = 0x05 + TYPE_IDSET int8 = 0x06 + TYPE_STRING int8 = 0x07 + TYPE_STRINGSET int8 = 0x08 +) + +func ExpectToken(reader io.Reader, token int16) (int16, error) { + var tk int16 + err := binary.Read(reader, binary.BigEndian, &tk) + if err != nil { + return 0, err + } + if tk != token { + return 0, errors.Errorf("expected token found %d", token) + } + return tk, nil +} + +// TOKEN_COLUMN_INFO message +// length (bytes) +// token 2 +// column count 2 +// +// (n) columns +// +// name length 1 +// name (from prev) +// data type 1 +// +// (optional) +// if type decimal +// scale 1 + +// note RelationName and AliasName members from +// PlannerColumn are not sent over the wire +func WriteSchema(schema types.Schema) ([]byte, error) { + buf := new(bytes.Buffer) + writer := bufio.NewWriter(buf) + // write token + writeToken(writer, TOKEN_SCHEMA_INFO) + + // column count + writeInt16(writer, int16(len(schema))) + + // for each column + for _, s := range schema { + // name len byte + writeInt8(writer, int8(len(s.ColumnName))) + // name + writer.WriteString(s.ColumnName) + // type byte + switch ty := s.Type.(type) { + case *parser.DataTypeID: + writeInt8(writer, TYPE_ID) + + case *parser.DataTypeBool: + writeInt8(writer, TYPE_BOOL) + + case *parser.DataTypeInt: + writeInt8(writer, TYPE_INT) + + case *parser.DataTypeDecimal: + writeInt8(writer, TYPE_DECIMAL) + writeInt8(writer, int8(ty.Scale)) + + case *parser.DataTypeTimestamp: + writeInt8(writer, TYPE_TIMESTAMP) + + case *parser.DataTypeIDSet: + writeInt8(writer, TYPE_IDSET) + + case *parser.DataTypeString: + writeInt8(writer, TYPE_STRING) + + case *parser.DataTypeStringSet: + writeInt8(writer, TYPE_STRINGSET) + + default: + return []byte{}, errors.Errorf("unexpected type '%T'", s.Type) + } + } + writer.Flush() + return buf.Bytes(), nil +} + +func ReadSchema(reader io.Reader) (types.Schema, error) { + var columnCount int16 + err := binary.Read(reader, binary.BigEndian, &columnCount) + if err != nil { + return nil, err + } + + var schema types.Schema + for i := 0; i < int(columnCount); i++ { + + var nameLen int8 + err = binary.Read(reader, binary.BigEndian, &nameLen) + if err != nil { + return nil, err + } + bname := make([]byte, nameLen) + err = binary.Read(reader, binary.BigEndian, &bname) + if err != nil { + return nil, err + } + colName := string(bname) + + var typ int8 + err = binary.Read(reader, binary.BigEndian, &typ) + if err != nil { + return nil, err + } + var dataType parser.ExprDataType + + switch typ { + case TYPE_ID: + dataType = parser.NewDataTypeID() + + case TYPE_BOOL: + dataType = parser.NewDataTypeBool() + + case TYPE_INT: + dataType = parser.NewDataTypeInt() + + case TYPE_DECIMAL: + var scale int8 + err = binary.Read(reader, binary.BigEndian, &scale) + if err != nil { + return nil, err + } + dataType = parser.NewDataTypeDecimal(int64(scale)) + + case TYPE_TIMESTAMP: + dataType = parser.NewDataTypeTimestamp() + + case TYPE_IDSET: + dataType = parser.NewDataTypeIDSet() + + case TYPE_STRING: + dataType = parser.NewDataTypeString() + + case TYPE_STRINGSET: + dataType = parser.NewDataTypeStringSet() + } + + schema = append(schema, &types.PlannerColumn{ + ColumnName: colName, + Type: dataType, + }) + } + return schema, nil +} + +// TOKEN_ROW message +// length (bytes) +// token 2 +// +// (n) columns +// if column length is 0 --> null +// +// - for ID, INT +// +// column length 1 +// value 8 +// +// - for DECIMAL +// +// column length 1 +// value 8 +// +// - for BOOL +// +// column length 1 +// value 1 +// +// - for TIMESTAMP +// +// column length 1 +// value 8 +// +// - for IDSET +// +// set length 2 +// (n) values +// value 8 +// +// - for STRING +// +// column length 2 +// value (from prev) +// +// - for STRINGSET +// +// set length 2 +// (n) values +// value len 2 +// value (from prev) +// + +func WriteRow(row types.Row, schema types.Schema) ([]byte, error) { + buf := new(bytes.Buffer) + writer := bufio.NewWriter(buf) + // write token + writeToken(writer, TOKEN_ROW) + + // for each column + for i, s := range schema { + val := row[i] + switch s.Type.(type) { + case *parser.DataTypeID, *parser.DataTypeInt: + if val == nil { + writeInt8(writer, 0) + } else { + writeInt8(writer, 8) + v, ok := row[i].(int64) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt64(writer, v) + } + + case *parser.DataTypeDecimal: + if val == nil { + writeInt8(writer, 0) + } else { + writeInt8(writer, 8) + v, ok := row[i].(pql.Decimal) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt64(writer, v.ToInt64(v.Scale)) + } + + case *parser.DataTypeBool: + if val == nil { + writeInt8(writer, 0) + } else { + writeInt8(writer, 1) + v, ok := row[i].(bool) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + if v { + writeInt8(writer, 1) + } else { + writeInt8(writer, 0) + } + } + + case *parser.DataTypeTimestamp: + if val == nil { + writeInt8(writer, 0) + } else { + writeInt8(writer, 8) + v, ok := row[i].(time.Time) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt64(writer, v.UnixNano()) + } + + case *parser.DataTypeIDSet: + if val == nil { + writeInt16(writer, 0) + } else { + v, ok := row[i].([]int64) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt16(writer, int16(len(v))) + for _, s := range v { + writeInt64(writer, s) + } + } + + case *parser.DataTypeString: + if val == nil { + writeInt16(writer, 0) + } else { + v, ok := row[i].(string) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt16(writer, int16(len(v))) + writer.WriteString(v) + } + + case *parser.DataTypeStringSet: + if val == nil { + writeInt16(writer, 0) + } else { + v, ok := row[i].([]string) + if !ok { + return []byte{}, errors.Errorf("unexpected type '%T'", row[i]) + } + writeInt16(writer, int16(len(v))) + for _, s := range v { + writeInt16(writer, int16(len(s))) + writer.WriteString(s) + } + } + + default: + return []byte{}, errors.Errorf("unexpected type '%T'", s.Type) + } + } + writer.Flush() + return buf.Bytes(), nil +} + +func ReadRow(reader io.Reader, schema types.Schema) (types.Row, error) { + + row := make(types.Row, len(schema)) + + for idx, s := range schema { + switch t := s.Type.(type) { + case *parser.DataTypeID, *parser.DataTypeInt: + var len int8 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + var value int64 + err := binary.Read(reader, binary.BigEndian, &value) + if err != nil { + return nil, err + } + row[idx] = value + } + + case *parser.DataTypeDecimal: + var len int8 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + var value int64 + err := binary.Read(reader, binary.BigEndian, &value) + if err != nil { + return nil, err + } + row[idx] = pql.NewDecimal(value, t.Scale) + } + + case *parser.DataTypeBool: + var len int8 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + var value int8 + err := binary.Read(reader, binary.BigEndian, &value) + if err != nil { + return nil, err + } + row[idx] = value == 1 + } + + case *parser.DataTypeTimestamp: + var len int8 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + var value int64 + err := binary.Read(reader, binary.BigEndian, &value) + if err != nil { + return nil, err + } + row[idx] = time.Unix(0, value) + } + + case *parser.DataTypeIDSet: + var len int16 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + set := make([]int64, len) + for j, _ := range set { + var value int64 + err := binary.Read(reader, binary.BigEndian, &value) + if err != nil { + return nil, err + } + set[j] = value + } + row[idx] = set + } + + case *parser.DataTypeString: + var len int16 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + bvalue := make([]byte, len) + err = binary.Read(reader, binary.BigEndian, &bvalue) + if err != nil { + return nil, err + } + row[idx] = string(bvalue) + } + + case *parser.DataTypeStringSet: + var len int16 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return nil, err + } + if len == 0 { + row[idx] = nil + } else { + set := make([]string, len) + for j, _ := range set { + var vlen int16 + err = binary.Read(reader, binary.BigEndian, &vlen) + if err != nil { + return nil, err + } + bvalue := make([]byte, vlen) + err = binary.Read(reader, binary.BigEndian, &bvalue) + if err != nil { + return nil, err + } + set[j] = string(bvalue) + } + row[idx] = set + } + + default: + return nil, errors.Errorf("unexpected type '%T'", s.Type) + } + } + + return row, nil +} + +// TOKEN_DONE message +// length (bytes) +// token 2 + +func WriteDone() []byte { + buf := new(bytes.Buffer) + writer := bufio.NewWriter(buf) + // write token + writeToken(writer, TOKEN_DONE) + writer.Flush() + return buf.Bytes() +} + +// TOKEN_ERROR_MESSAGE message +// length (bytes) +// token 2 +// +// message len 4 +// message (from prev) + +func WriteError(err error) []byte { + buf := new(bytes.Buffer) + writer := bufio.NewWriter(buf) + // write token + writeToken(writer, TOKEN_ERROR_MESSAGE) + // write error len + t := err.Error() + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, uint32(len(t))) + writer.Write(b) + // write error + writer.WriteString(t) + writer.Flush() + + return buf.Bytes() +} + +func ReadError(reader io.Reader) (string, error) { + var len int16 + err := binary.Read(reader, binary.BigEndian, &len) + if err != nil { + return "", err + } + bvalue := make([]byte, len) + err = binary.Read(reader, binary.BigEndian, &bvalue) + if err != nil { + return "", err + } + return string(bvalue), nil +} + +func writeToken(w io.Writer, token int16) { + writeInt16(w, token) +} + +func writeInt8(w io.Writer, i int8) { + b := make([]byte, 1) + b[0] = byte(i) + w.Write(b) +} + +func writeInt16(w io.Writer, i int16) { + b := make([]byte, 2) + binary.BigEndian.PutUint16(b, uint16(i)) + w.Write(b) +} + +func writeInt64(w io.Writer, i int64) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(i)) + w.Write(b) +} diff --git a/wireprotocol/wireprimitives_test.go b/wireprotocol/wireprimitives_test.go new file mode 100644 index 000000000..364430117 --- /dev/null +++ b/wireprotocol/wireprimitives_test.go @@ -0,0 +1,144 @@ +package wireprotocol_test + +import ( + "bytes" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/molecula/featurebase/v3/pql" + "github.com/molecula/featurebase/v3/sql3/parser" + "github.com/molecula/featurebase/v3/sql3/planner/types" + "github.com/molecula/featurebase/v3/wireprotocol" +) + +func TestWireProtocol_Schema(t *testing.T) { + + s := types.Schema{ + &types.PlannerColumn{ + ColumnName: "col1", + Type: parser.NewDataTypeID(), + }, + &types.PlannerColumn{ + ColumnName: "col2", + Type: parser.NewDataTypeInt(), + }, + &types.PlannerColumn{ + ColumnName: "col3", + Type: parser.NewDataTypeDecimal(4), + }, + &types.PlannerColumn{ + ColumnName: "col4", + Type: parser.NewDataTypeString(), + }, + &types.PlannerColumn{ + ColumnName: "col5", + Type: parser.NewDataTypeStringSet(), + }, + &types.PlannerColumn{ + ColumnName: "col6", + Type: parser.NewDataTypeIDSet(), + }, + &types.PlannerColumn{ + ColumnName: "col7", + Type: parser.NewDataTypeBool(), + }, + &types.PlannerColumn{ + ColumnName: "col8", + Type: parser.NewDataTypeTimestamp(), + }, + } + + b, err := wireprotocol.WriteSchema(s) + if err != nil { + t.Fatal(err) + } + + rdr := bytes.NewReader(b) + _, err = wireprotocol.ExpectToken(rdr, wireprotocol.TOKEN_SCHEMA_INFO) + if err != nil { + t.Fatal(err) + } + + sr, err := wireprotocol.ReadSchema(rdr) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(s, sr); diff != "" { + t.Fatal(diff) + } +} + +func TestWireProtocol_Row(t *testing.T) { + + s := types.Schema{ + &types.PlannerColumn{ + ColumnName: "col1", + Type: parser.NewDataTypeID(), + }, + &types.PlannerColumn{ + ColumnName: "col2", + Type: parser.NewDataTypeInt(), + }, + &types.PlannerColumn{ + ColumnName: "col3", + Type: parser.NewDataTypeDecimal(4), + }, + &types.PlannerColumn{ + ColumnName: "col4", + Type: parser.NewDataTypeString(), + }, + &types.PlannerColumn{ + ColumnName: "col5", + Type: parser.NewDataTypeStringSet(), + }, + &types.PlannerColumn{ + ColumnName: "col6", + Type: parser.NewDataTypeIDSet(), + }, + &types.PlannerColumn{ + ColumnName: "col7", + Type: parser.NewDataTypeBool(), + }, + &types.PlannerColumn{ + ColumnName: "col8", + Type: parser.NewDataTypeTimestamp(), + }, + } + + r := types.Row{ + int64(1), + int64(2), + pql.NewDecimal(123400, 4), + string("foo"), + []string{"bar", "baz"}, + []int64{10, 20}, + bool(false), + time.Now().UTC(), + } + + b, err := wireprotocol.WriteRow(r, s) + if err != nil { + t.Fatal(err) + } + + rdr := bytes.NewReader(b) + _, err = wireprotocol.ExpectToken(rdr, wireprotocol.TOKEN_ROW) + if err != nil { + t.Fatal(err) + } + + rr, err := wireprotocol.ReadRow(rdr, s) + if err != nil { + t.Fatal(err) + } + + opt := cmp.Comparer(func(x, y pql.Decimal) bool { + return x.EqualTo(y) + }) + + if diff := cmp.Diff(r, rr, opt); diff != "" { + t.Fatal(diff) + } +}