scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

introduce performance counters and system table fanout, plus refactor metrics (#2363)

Browse source 
* performance counters

* first cut of perf counters and system table fanout and a wire protocol
* significantly refactored prometheus support; removed statsd and exprvar

* removed node_id

* put dax subquery test back

* Change Translator.TranslateFieldIDs method to take a dax.TableKeyer

There are a bunch of other calls to the Translator interface methods
with currently take an `index string`, and those need to be converted to
dax.TableKeyer as well. But I need to review each call, because in at
least one place I noticed one being called with `result.Index` instead
of with the qtbl available. And I don't yet know how those could be
different.

Co-authored-by: Travis Turner <travis@molecula.com>
This commit is contained in:
pokeeffe-molecula 2023-01-11 20:37:00 -06:00 • committed by GitHub
parent 17cb2cbb77
commit 7f6ea0e6e5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
48 changed files with 3074 additions and 1530 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
api.go
View file

@ -28,12 +28,12 @@ import (
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/rbf"
"github.com/prometheus/client_golang/prometheus"
//"github.com/molecula/featurebase/v3/pg"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
planner_types "github.com/molecula/featurebase/v3/sql3/planner/types"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/tracing"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
@ -277,7 +277,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
return nil, errors.Wrap(err, "creating index")
}
api.holder.Stats.Count(MetricCreateIndex, 1, 1.0)
CounterCreateIndex.Inc()
return index, nil
}
@ -318,7 +318,7 @@ func (api *API) DeleteDataframe(ctx context.Context, indexName string) error {
api.server.logger.Errorf("problem sending DeleteIndex message: %s", err)
return errors.Wrap(err, "sending DeleteIndex message")
}
api.holder.Stats.Count(MetricDeleteDataframe, 1, 1.0)
CounterDeleteDataframe.Inc()
return nil
}
@ -353,7 +353,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
return errors.Wrap(err, "deleting id allocation for index")
}
}
api.holder.Stats.Count(MetricDeleteIndex, 1, 1.0)
CounterDeleteIndex.Inc()
return nil
}
@ -405,7 +405,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
return nil, errors.Wrap(err, "sending CreateField message")
}
api.holder.Stats.CountWithCustomTags(MetricCreateField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
CounterCreateField.With(prometheus.Labels{"index": indexName})
return field, nil
}
@ -749,7 +749,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str
api.server.logger.Errorf("problem sending DeleteField message: %s", err)
return errors.Wrap(err, "sending DeleteField message")
}
api.holder.Stats.CountWithCustomTags(MetricDeleteField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
CounterDeleteField.With(prometheus.Labels{"index": indexName})
return nil
}
@ -781,7 +781,7 @@ func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName str
api.server.logger.Errorf("problem sending DeleteAvailableShard message: %s", err)
return errors.Wrap(err, "sending DeleteAvailableShard message")
}
api.holder.Stats.CountWithCustomTags(MetricDeleteAvailableShard, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
CounterDeleteAvailableShard.With(prometheus.Labels{"index": indexName}).Inc()
return nil
}
@ -2095,15 +2095,6 @@ func (api *API) AvailableShards(ctx context.Context, indexName string) (*roaring
return index.AvailableShards(false), nil
}
// StatsWithTags returns an instance of whatever implementation of StatsClient
// pilosa is using with the given tags.
func (api *API) StatsWithTags(tags []string) stats.StatsClient {
if api.holder == nil || api.cluster == nil {
return nil
}
return api.holder.Stats.WithTags(tags...)
}
// LongQueryTime returns the configured threshold for logging/statting
// long running queries.
func (api *API) LongQueryTime() time.Duration {
@ -2390,19 +2381,19 @@ func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Du
switch err {
case nil:
if exclusive {
api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0)
CounterExclusiveTransactionRequest.Inc()
} else {
api.holder.Stats.Count(MetricTransactionStart, 1, 1.0)
CounterTransactionStart.Inc()
}
case ErrTransactionExclusive:
if exclusive {
api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0)
CounterExclusiveTransactionBlocked.Inc()
} else {
api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0)
CounterTransactionBlocked.Inc()
}
}
if exclusive && t != nil && t.Active {
api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
CounterExclusiveTransactionActive.Inc()
}
return t, err
}
@ -2414,9 +2405,9 @@ func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) (
t, err := api.server.FinishTransaction(ctx, id, remote)
if err == nil {
if t.Exclusive {
api.holder.Stats.Count(MetricExclusiveTransactionEnd, 1, 1.0)
CounterExclusiveTransactionEnd.Inc()
} else {
api.holder.Stats.Count(MetricTransactionEnd, 1, 1.0)
CounterTransactionEnd.Inc()
}
}
return t, err
@ -2436,7 +2427,7 @@ func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Tr
t, err := api.server.GetTransaction(ctx, id, remote)
if err == nil {
if t.Exclusive && t.Active {
api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
CounterExclusiveTransactionActive.Inc()
}
}
return t, err
@ -3045,6 +3036,10 @@ func (api *API) CompilePlan(ctx context.Context, q string) (planner_types.PlanOp
return api.server.CompileExecutionPlan(ctx, q)
}
func (api *API) RehydratePlanOperator(ctx context.Context, reader io.Reader) (planner_types.PlanOperator, error) {
return api.server.RehydratePlanOperator(ctx, reader)
}
func (api *API) RBFDebugInfo() map[string]*rbf.DebugInfo {
infos := make(map[string]*rbf.DebugInfo)
@ -3342,6 +3337,7 @@ type SystemAPI interface {
ClusterState() string
DataDir() string
NodeID() string
ClusterNodes() []ClusterNode
}
@ -3413,6 +3409,10 @@ func (fsapi *FeatureBaseSystemAPI) DataDir() string {
return fsapi.server.dataDir
}
func (fsapi *FeatureBaseSystemAPI) NodeID() string {
return fsapi.cluster.Node.ID
}
func (fsapi *FeatureBaseSystemAPI) ClusterNodes() []ClusterNode {
result := make([]ClusterNode, 0)

8
batch/batch.go
View file

@ -755,7 +755,7 @@ func (b *Batch) Import() error {
}()
}
defer func() {
b.importer.StatsTiming(MetricBatchImportDurationSeconds, time.Since(start), 1.0)
featurebase.SummaryBatchImportDurationSeconds.Observe(time.Since(start).Seconds())
}()
size := len(b.ids)
@ -828,7 +828,7 @@ func (b *Batch) Flush() error {
if err != nil {
b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
}
b.importer.StatsTiming(MetricBatchFlushDurationSeconds, time.Since(start), 1.0)
featurebase.SummaryBatchFlushDurationSeconds.Observe(time.Since(start).Seconds())
}()
importStart := time.Now()
@ -1188,7 +1188,7 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error {
}
}
b.importer.StatsTiming(MetricBatchShardImportBuildRequestsSeconds, time.Since(start), 1.0)
featurebase.SummaryBatchShardImportBuildRequestsSeconds.Observe(time.Since(start).Seconds())
start = time.Now()
eg := egpool.Group{PoolSize: 20}
for shard, request := range requests {
@ -1200,7 +1200,7 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error {
}
err := eg.Wait()
dur := time.Since(start)
b.importer.StatsTiming(MetricBatchShardImportDurationSeconds, dur, 1.0)
featurebase.SummaryBatchImportDurationSeconds.Observe(dur.Seconds())
b.log.Printf("import shard took: %v\n", dur)
return errors.Wrap(err, "doing shard-transactional imports")
}

25
batch/metrics.go
View file

@ -1,26 +1 @@
package batch
const (
// MetricBatchImportDurationSeconds records the full time of the
// RecordBatch.Import call. This includes starting and finishing a
// transaction, doing key translation, building fragments locally,
// importing all data, and resetting internal structures.
MetricBatchImportDurationSeconds = "batch_import_duration_seconds"
// MetricBatchFlushDurationSeconds records the full time for
// RecordBatch.Flush (if splitBatchMode is in use). This includes
// starting and finishing a transaction, importing all data, and
// resetting internal structures.
MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds"
// MetricBatchShardImportBuildRequestsSeconds is the time it takes
// after making fragments to build the shard-transactional request
// objects (but not actually import them or do any network activity).
MetricBatchShardImportBuildRequestsSeconds = "batch_shard_import_build_requests_seconds"
// MetricBatchShardImportDurationSeconds is the time it takes to
// import all data for all shards in the batch using the
// shard-transactional endpoint. This does not include the time it
// takes to build the requests locally.
MetricBatchShardImportDurationSeconds = "batch_shard_import_duration_seconds"
)

51
cache.go
View file

@ -12,7 +12,6 @@ import (
"github.com/molecula/featurebase/v3/lru"
pb "github.com/molecula/featurebase/v3/proto"
"github.com/molecula/featurebase/v3/stats"
"github.com/pkg/errors"
)
@ -40,9 +39,6 @@ type cache interface {
// Returns an ordered list of the top ranked bitmaps.
Top() []bitmapPair
// SetStats defines the stats client used in the cache.
SetStats(s stats.StatsClient)
// Clear removes everything from the cache. If possible it should leave allocated structures in place to be reused.
Clear()
}
@ -51,7 +47,6 @@ type cache interface {
type lruCache struct {
cache *lru.Cache
counts map[uint64]uint64
stats stats.StatsClient
// maxEntries is saved to support Clear which recreates the cache.
maxEntries uint32
}
@ -61,7 +56,6 @@ func newLRUCache(maxEntries uint32) *lruCache {
c := &lruCache{
cache: lru.New(int(maxEntries)),
counts: make(map[uint64]uint64),
stats: stats.NopStatsClient,
maxEntries: maxEntries,
}
c.cache.OnEvicted = c.onEvicted
@ -119,11 +113,6 @@ func (c *lruCache) Top() []bitmapPair {
return a
}
// SetStats defines the stats client used in the cache.
func (c *lruCache) SetStats(s stats.StatsClient) {
c.stats = s
}
func (c *lruCache) Clear() {
for k := range c.counts {
delete(c.counts, k)
@ -157,8 +146,6 @@ type rankCache struct {
// thresholdValue is the value of the last item in the cache
thresholdValue uint64
stats stats.StatsClient
}
// NewRankCache returns a new instance of RankCache.
@ -167,7 +154,6 @@ func NewRankCache(maxEntries uint32) *rankCache {
maxEntries: maxEntries,
thresholdBuffer: int(thresholdFactor * float64(maxEntries)),
entries: make(map[uint64]uint64),
stats: stats.NopStatsClient,
}
}
@ -228,7 +214,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) {
// as this can take up an upbounded amount of memory. This is especially
// true when restoring shards as all rows will be added.
if len(c.entries) > int(2*c.maxEntries) {
c.stats.Count(MetricRecalculateCache, 1, 1.0)
CounterRecalculateCache.Inc()
c.recalculate()
}
}
@ -273,7 +259,7 @@ func (c *rankCache) Invalidate() {
func (c *rankCache) Recalculate() {
c.mu.Lock()
defer c.mu.Unlock()
c.stats.Count(MetricRecalculateCache, 1, 1.0)
CounterRecalculateCache.Inc()
c.recalculate()
}
@ -285,12 +271,12 @@ func (c *rankCache) invalidate() {
// This is somewhat necessary for now since recalculation is not cheap.
// The cache will remain flagged as dirty and will be recalculated if Top is called.
// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0)
CounterInvalidateCacheSkipped.Inc()
// Ensure that we're marked as dirty even if we weren't otherwise.
c.dirty = true
return
}
c.stats.Count(MetricInvalidateCache, 1, 1.0)
CounterInvalidateCache.Inc()
c.recalculate()
}
@ -316,7 +302,7 @@ func (c *rankCache) recalculate() {
// Store the count of the item at the threshold index.
length := len(c.rankings)
c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
GaugeRankCacheLength.Set(float64(length))
var removeItems []bitmapPair // cached, ordered list
if length > int(c.maxEntries) {
@ -332,7 +318,7 @@ func (c *rankCache) recalculate() {
// If size is larger than the threshold then trim it.
if len(c.entries) > c.thresholdBuffer {
c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
CounterCacheThresholdReached.Inc()
for _, pair := range removeItems {
delete(c.entries, pair.ID)
}
@ -342,11 +328,6 @@ func (c *rankCache) recalculate() {
c.dirty = false
}
// SetStats defines the stats client used in the cache.
func (c *rankCache) SetStats(s stats.StatsClient) {
c.stats = s
}
// Top returns an ordered list of pairs.
func (c *rankCache) Top() []bitmapPair {
c.mu.Lock()
@ -354,7 +335,7 @@ func (c *rankCache) Top() []bitmapPair {
if c.dirty {
// The cache is dirty, so we need to recalculate it to get a consistent view.
c.stats.Count(MetricReadDirtyCache, 1, 1.0)
CounterReadDirtyCache.Inc()
c.recalculate()
}
@ -605,25 +586,21 @@ func (p uint64Slice) Len() int { return len(p) }
func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }
// nopCache represents a no-op Cache implementation.
type nopCache struct {
stats stats.StatsClient
}
type nopCache struct{}
// Ensure NopCache implements Cache.
var globalNopCache cache = nopCache{
stats: stats.NopStatsClient,
}
var globalNopCache cache = nopCache{}
func (c nopCache) Add(uint64, uint64) {}
func (c nopCache) BulkAdd(uint64, uint64) {}
func (c nopCache) Get(uint64) uint64 { return 0 }
func (c nopCache) IDs() []uint64 { return []uint64{} }
func (c nopCache) Invalidate() {}
func (c nopCache) Len() int { return 0 }
func (c nopCache) Recalculate() {}
func (c nopCache) SetStats(stats.StatsClient) {}
func (c nopCache) Clear() {}
func (c nopCache) Invalidate() {}
func (c nopCache) Len() int { return 0 }
func (c nopCache) Recalculate() {}
func (c nopCache) Clear() {}
func (c nopCache) Top() []bitmapPair {
return []bitmapPair{}

16
client/client.go
View file

@ -29,7 +29,6 @@ import (
"github.com/molecula/featurebase/v3/pb"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/vprint"
"github.com/opentracing/opentracing-go"
"github.com/pkg/errors"
@ -52,7 +51,6 @@ type Client struct {
manualFragmentNode *fragmentNode
manualServerURI *pnet.URI
tracer opentracing.Tracer
Stats stats.StatsClient
// An exponential backoff algorithm retries requests exponentially (if an HTTP request fails),
// increasing the waiting time between retries up to a maximum backoff time.
maxBackoff time.Duration
@ -211,11 +209,6 @@ func newClientWithOptions(options *ClientOptions) *Client {
} else {
c.tracer = options.tracer
}
if options.stats == nil {
c.Stats = stats.NopStatsClient
} else {
c.Stats = options.stats
}
c.maxRetries = *options.retries
c.maxBackoff = 2 * time.Minute
@ -1359,7 +1352,6 @@ type ClientOptions struct {
manualServerAddress bool
tracer opentracing.Tracer
retries *int
stats stats.StatsClient
nat map[pnet.URI]pnet.URI
pathPrefix string
}
@ -1445,14 +1437,6 @@ func OptClientRetries(retries int) ClientOption {
}
}
// OptClientStatsClient sets a stats client, such as Prometheus
func OptClientStatsClient(stats stats.StatsClient) ClientOption {
return func(options *ClientOptions) error {
options.stats = stats
return nil
}
}
// OptClientNAT sets a NAT map used to translate the advertised URI to something
// else (for example, when accessing pilosa running in docker).
func OptClientNAT(nat map[string]string) ClientOption {

4
client/importer.go
View file

@ -328,7 +328,3 @@ func (i *importer) EncodeImport(ctx context.Context, tid dax.TableID, fld *dax.F
func (i *importer) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, path string, data []byte) error {
return i.client.DoImport(string(tid), shard, path, data)
}
func (i *importer) StatsTiming(name string, value time.Duration, rate float64) {
i.client.Stats.Timing(name, value, rate)
}

103
dax/queryer/orchestrator.go
View file

@ -14,8 +14,8 @@ import (
"github.com/molecula/featurebase/v3/errors"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/tracing"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/sync/errgroup"
)
@ -74,7 +74,7 @@ type Translator interface {
// TODO(jaffee) the naming here is a cluster. TranslateIndexIDs takes a list, but TranslateFieldIDs takes a set, both have alternate methods that take the other thing. :facepalm:
TranslateIndexIDs(ctx context.Context, index string, ids []uint64) ([]string, error)
TranslateIndexIDSet(ctx context.Context, index string, ids map[uint64]struct{}) (map[uint64]string, error)
TranslateFieldIDs(ctx context.Context, index, field string, ids map[uint64]struct{}) (map[uint64]string, error)
TranslateFieldIDs(ctx context.Context, tableKeyer dax.TableKeyer, field string, ids map[uint64]struct{}) (map[uint64]string, error)
TranslateFieldListIDs(ctx context.Context, index, field string, ids []uint64) ([]string, error)
}
@ -87,7 +87,6 @@ type orchestrator struct {
// Client used for remote requests.
client *featurebase.InternalClient
stats stats.StatsClient
logger logger.Logger
}
@ -433,11 +432,11 @@ func (o *orchestrator) executeCall(ctx context.Context, tableKeyer dax.TableKeye
} else if err := o.validateCallArgs(c); err != nil {
return nil, errors.Wrap(err, "validating args")
}
indexTag := "index:" + string(tableKeyer.Key())
metricName := "query_" + strings.ToLower(c.Name) + "_total"
statFn := func() {
labels := prometheus.Labels{"index": string(tableKeyer.Key())}
statFn := func(ctr *prometheus.CounterVec) {
if !opt.Remote {
o.stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag})
ctr.With(labels).Inc()
}
}
@ -449,101 +448,106 @@ func (o *orchestrator) executeCall(ctx context.Context, tableKeyer dax.TableKeye
switch c.Name {
case "Sum":
statFn()
statFn(featurebase.CounterQuerySumTotal)
res, err := o.executeSum(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeSum")
case "Min":
statFn()
statFn(featurebase.CounterQueryMinTotal)
res, err := o.executeMin(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeMin")
case "Max":
statFn()
statFn(featurebase.CounterQueryMaxTotal)
res, err := o.executeMax(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeMax")
case "MinRow":
statFn()
statFn(featurebase.CounterQueryMinRowTotal)
res, err := o.executeMinRow(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeMinRow")
case "MaxRow":
statFn()
statFn(featurebase.CounterQueryMaxRowTotal)
res, err := o.executeMaxRow(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeMaxRow")
// case "Clear":
// statFn()
// statFn(featurebase.CounterQueryClearTotal)
// res, err := o.executeClearBit(ctx, index, c, opt)
// return res, errors.Wrap(err, "executeClearBit")
// case "ClearRow":
// statFn()
// statFn(featurebase.CounterQueryClearRowTotal)
// res, err := o.executeClearRow(ctx, index, c, shards, opt)
// return res, errors.Wrap(err, "executeClearRow")
case "Distinct":
statFn()
statFn(featurebase.CounterQueryDistinctTotal)
res, err := o.executeDistinct(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeDistinct")
// case "Store":
// statFn()
// statFn(featurebase.CounterQueryStoreTotal)
// res, err := o.executeSetRow(ctx, index, c, shards, opt)
// return res, errors.Wrap(err, "executeSetRow")
case "Count":
statFn()
statFn(featurebase.CounterQueryCountTotal)
res, err := o.executeCount(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeCount")
// case "Set":
// statFn()
// statFn(featurebase.CounterQuerySetTotal)
// res, err := o.executeSet(ctx, index, c, opt)
// return res, errors.Wrap(err, "executeSet")
case "TopK":
statFn()
statFn(featurebase.CounterQueryTopKTotal)
res, err := o.executeTopK(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeTopK")
case "TopN":
statFn()
statFn(featurebase.CounterQueryTopNTotal)
res, err := o.executeTopN(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeTopN")
case "Rows":
statFn()
statFn(featurebase.CounterQueryRowsTotal)
res, err := o.executeRows(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeRows")
case "Extract":
statFn()
statFn(featurebase.CounterQueryExtractTotal)
res, err := o.executeExtract(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeExtract")
case "GroupBy":
statFn()
statFn(featurebase.CounterQueryGroupByTotal)
res, err := o.executeGroupBy(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeGroupBy")
case "Options":
statFn()
statFn(featurebase.CounterQueryOptionsTotal)
res, err := o.executeOptionsCall(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeOptionsCall")
case "IncludesColumn":
statFn(featurebase.CounterQueryIncludesColumnTotal)
res, err := o.executeIncludesColumnCall(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeIncludesColumnCall")
case "FieldValue":
statFn()
statFn(featurebase.CounterQueryFieldValueTotal)
res, err := o.executeFieldValueCall(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeFieldValueCall")
case "Precomputed":
statFn(featurebase.CounterQueryPrecomputedTotal)
res, err := o.executePrecomputedCall(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executePrecomputedCall")
case "UnionRows":
statFn(featurebase.CounterQueryUnionRowsTotal)
res, err := o.executeUnionRows(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeUnionRows")
case "ConstRow":
statFn(featurebase.CounterQueryConstRowTotal)
res, err := o.executeConstRow(ctx, tableKeyer, c)
return res, errors.Wrap(err, "executeConstRow")
case "Limit":
statFn(featurebase.CounterQueryLimitTotal)
res, err := o.executeLimitCall(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeLimitCall")
case "Percentile":
statFn(featurebase.CounterQueryPercentileTotal)
res, err := o.executePercentile(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executePercentile")
// case "Delete":
// statFn() //TODO(twg) need this?
// statFn(featurebase.CounterQueryDeleteTotal)
// res, err := o.executeDeleteRecords(ctx, index, c, shards, opt)
// return res, errors.Wrap(err, "executeDelete")
default: // o.g. "Row", "Union", "Intersect" or anything that returns a bitmap.
statFn()
res, err := o.executeBitmapCall(ctx, tableKeyer, c, shards, opt)
return res, errors.Wrap(err, "executeBitmapCall")
}
@ -1116,13 +1120,42 @@ func (o *orchestrator) executeBitmapCall(ctx context.Context, tableKeyer dax.Tab
span.LogKV("pqlCallName", c.Name)
defer span.Finish()
indexTag := "index:" + string(tableKeyer.Key())
metricName := "query_" + strings.ToLower(c.Name) + "_total"
if c.Name == "Row" && c.HasConditionArg() {
metricName = "query_row_bsi_total"
labels := prometheus.Labels{"index": string(tableKeyer.Key())}
statFn := func(ctr *prometheus.CounterVec) {
if !opt.Remote {
ctr.With(labels).Inc()
}
}
if !opt.Remote {
o.stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag})
switch c.Name {
case "Row":
if c.HasConditionArg() {
statFn(featurebase.CounterQueryRowBSITotal)
} else {
statFn(featurebase.CounterQueryRowTotal)
}
case "Range":
statFn(featurebase.CounterQueryRangeTotal)
case "Difference":
statFn(featurebase.CounterQueryBitmapTotal)
case "Intersect":
statFn(featurebase.CounterQueryIntersectTotal)
case "Union":
statFn(featurebase.CounterQueryUnionTotal)
case "InnerUnionRows":
statFn(featurebase.CounterQueryInnerUnionRowsTotal)
case "Xor":
statFn(featurebase.CounterQueryXorTotal)
case "Not":
statFn(featurebase.CounterQueryNotTotal)
case "Shift":
statFn(featurebase.CounterQueryShiftTotal)
case "All":
statFn(featurebase.CounterQueryAllTotal)
default:
statFn(featurebase.CounterQueryBitmapTotal)
}
}
// Merge returned results at coordinating node.
@ -2985,9 +3018,7 @@ func (o *orchestrator) preTranslateMatrixSet(ctx context.Context, mat featurebas
}
}
index := string(tableKeyer.Key())
return o.trans.TranslateFieldIDs(ctx, index, field, ids)
return o.trans.TranslateFieldIDs(ctx, tableKeyer, field, ids)
}
func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedTable, call *pql.Call, result interface{}, idSet map[uint64]string) (_ interface{}, err error) {
@ -3164,7 +3195,7 @@ func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedT
fieldTranslations := make(map[string]map[uint64]string)
for field, ids := range fieldIDs {
trans, err := o.trans.TranslateFieldIDs(ctx, idx.Name, field.Name, ids)
trans, err := o.trans.TranslateFieldIDs(ctx, qtbl, field.Name, ids)
if err != nil {
return nil, errors.Wrapf(err, "translating IDs in field '%q'", field.Name)
}

2
dax/queryer/queryer.go
View file

@ -22,7 +22,6 @@ import (
"github.com/molecula/featurebase/v3/sql3/parser"
"github.com/molecula/featurebase/v3/sql3/planner"
plannertypes "github.com/molecula/featurebase/v3/sql3/planner/types"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/systemlayer"
uuid "github.com/satori/go.uuid"
)
@ -89,7 +88,6 @@ func (q *Queryer) Orchestrator(qual dax.TableQualifier) *qualifiedOrchestrator {
topology: &MDSTopology{noder: q.noder},
// TODO(jaffee) using default http.Client probably bad... need to set some timeouts.
client: q.fbClient,
stats: stats.NopStatsClient,
logger: q.logger,
}

10
dax/queryer/translator.go
View file

@ -229,15 +229,19 @@ func (m *mdsTranslator) TranslateIndexIDSet(ctx context.Context, table string, i
}
return ret, nil
}
func (m *mdsTranslator) TranslateFieldIDs(ctx context.Context, table, field string, ids map[uint64]struct{}) (map[uint64]string, error) {
func (m *mdsTranslator) TranslateFieldIDs(ctx context.Context, tableKeyer dax.TableKeyer, field string, ids map[uint64]struct{}) (map[uint64]string, error) {
idList := make([]uint64, 0, len(ids))
for id := range ids {
idList = append(idList, id)
}
stringList, err := m.TranslateFieldListIDs(ctx, table, field, idList)
// TODO(tlt): convert TranslateFieldListIDs (and the other Translator
// interface methods) to TableKeyer.
index := string(tableKeyer.Key())
stringList, err := m.TranslateFieldListIDs(ctx, index, field, idList)
if err != nil {
return nil, errors.Wrapf(err, "translating field ids on field: %s, %s", table, field)
return nil, errors.Wrapf(err, "translating field ids on field: %s, %s", tableKeyer, field)
}
ret := make(map[uint64]string)

1
dax/test/dax/dax_test.go
View file

@ -112,7 +112,6 @@ func TestDAXIntegration(t *testing.T) {
"alterTable/alterTableBadTable", // looks like table does not exist is a different error in DAX
"top-tests/test-1", // don't know why this is failing at all
"delete_tests",
"subquerytable", // subqueries seem to be a problem
}
doSkip := func(name string) bool {

104
executor.go
View file

@ -28,6 +28,7 @@ import (
"github.com/molecula/featurebase/v3/testhook"
"github.com/molecula/featurebase/v3/tracing"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/sync/errgroup"
)
@ -164,16 +165,16 @@ func (e *executor) Close() error {
// PoolSize is exported to let the task pool update us
func (e *executor) PoolSize(n int) {
if e.Holder != nil {
e.Holder.Stats.Gauge("worker_total", float64(n), 0)
GaugeWorkerTotal.Set(float64(n))
}
}
// InitStats initializes stats counters. Must be called after Holder set.
func (e *executor) InitStats() {
if e.Holder != nil {
e.Holder.Stats.Count("job_total", 0, 0)
CounterJobTotal.Add(0)
l, _, _ := e.workers.Stats()
e.Holder.Stats.Gauge("worker_total", float64(l), 0)
GaugeWorkerTotal.Set(float64(l))
}
}
@ -683,11 +684,11 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p
} else if err := e.validateCallArgs(c); err != nil {
return nil, errors.Wrap(err, "validating args")
}
indexTag := "index:" + index
metricName := "query_" + strings.ToLower(c.Name) + "_total"
statFn := func() {
labels := prometheus.Labels{"index": index}
statFn := func(ctr *prometheus.CounterVec) {
if !opt.Remote {
e.Holder.Stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag})
ctr.With(labels).Inc()
}
}
@ -719,114 +720,122 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p
switch c.Name {
case "Sum":
statFn()
statFn(CounterQuerySumTotal)
res, err := e.executeSum(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeSum")
case "Min":
statFn()
statFn(CounterQueryMinTotal)
res, err := e.executeMin(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeMin")
case "Max":
statFn()
statFn(CounterQueryMaxTotal)
res, err := e.executeMax(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeMax")
case "MinRow":
statFn()
statFn(CounterQueryMinRowTotal)
res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeMinRow")
case "MaxRow":
statFn()
statFn(CounterQueryMaxRowTotal)
res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeMaxRow")
case "Clear":
statFn()
statFn(CounterQueryClearTotal)
res, err := e.executeClearBit(ctx, qcx, index, c, opt)
return res, errors.Wrap(err, "executeClearBit")
case "ClearRow":
statFn()
statFn(CounterQueryClearRowTotal)
res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeClearRow")
case "Distinct":
statFn()
statFn(CounterQueryDistinctTotal)
res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeDistinct")
case "Store":
statFn()
statFn(CounterQueryStoreTotal)
res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeSetRow")
case "Count":
statFn()
statFn(CounterQueryCountTotal)
res, err := e.executeCount(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeCount")
case "Set":
statFn()
statFn(CounterQuerySetTotal)
res, err := e.executeSet(ctx, qcx, index, c, opt)
return res, errors.Wrap(err, "executeSet")
case "TopK":
statFn()
statFn(CounterQueryTopKTotal)
res, err := e.executeTopK(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeTopK")
case "TopN":
statFn()
statFn(CounterQueryTopNTotal)
res, err := e.executeTopN(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeTopN")
case "Rows":
statFn()
statFn(CounterQueryRowsTotal)
res, err := e.executeRows(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeRows")
case "ExternalLookup":
statFn()
statFn(CounterQueryExternalLookupTotal)
res, err := e.executeExternalLookup(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeExternalLookup")
case "Extract":
statFn()
statFn(CounterQueryExtractTotal)
res, err := e.executeExtract(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeExtract")
case "GroupBy":
statFn()
statFn(CounterQueryGroupByTotal)
res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeGroupBy")
case "Options":
statFn()
statFn(CounterQueryOptionsTotal)
res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeOptionsCall")
case "IncludesColumn":
statFn(CounterQueryIncludesColumnTotal)
res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeIncludesColumnCall")
case "FieldValue":
statFn()
statFn(CounterQueryFieldValueTotal)
res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeFieldValueCall")
case "Precomputed":
statFn(CounterQueryPrecomputedTotal)
res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executePrecomputedCall")
case "UnionRows":
statFn(CounterQueryUnionRowsTotal)
res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeUnionRows")
case "ConstRow":
statFn(CounterQueryConstRowTotal)
res, err := e.executeConstRow(ctx, index, c)
return res, errors.Wrap(err, "executeConstRow")
case "Limit":
statFn(CounterQueryLimitTotal)
res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeLimitCall")
case "Percentile":
statFn(CounterQueryPercentileTotal)
res, err := e.executePercentile(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executePercentile")
case "Delete":
statFn() // TODO(twg) need this?
statFn(CounterQueryDeleteTotal)
res, err := e.executeDeleteRecords(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeDelete")
case "Sort":
statFn(CounterQuerySortTotal)
res, err := e.executeSort(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeSort")
case "Apply":
statFn(CounterQueryApplyTotal)
res, err := e.executeApply(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeApply")
case "Arrow":
statFn(CounterQueryArrowTotal)
res, err := e.executeArrow(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeArrow")
default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap.
statFn()
res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt)
return res, errors.Wrap(err, "executeBitmapCall")
}
@ -1505,13 +1514,42 @@ func (e *executor) executeBitmapCall(ctx context.Context, qcx *Qcx, index string
span.LogKV("pqlCallName", c.Name)
defer span.Finish()
indexTag := "index:" + index
metricName := "query_" + strings.ToLower(c.Name) + "_total"
if c.Name == "Row" && c.HasConditionArg() {
metricName = "query_row_bsi_total"
labels := prometheus.Labels{"index": index}
statFn := func(ctr *prometheus.CounterVec) {
if !opt.Remote {
ctr.With(labels).Inc()
}
}
if !opt.Remote {
e.Holder.Stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag})
switch c.Name {
case "Row":
if c.HasConditionArg() {
statFn(CounterQueryRowBSITotal)
} else {
statFn(CounterQueryRowTotal)
}
case "Range":
statFn(CounterQueryRangeTotal)
case "Difference":
statFn(CounterQueryBitmapTotal)
case "Intersect":
statFn(CounterQueryIntersectTotal)
case "Union":
statFn(CounterQueryUnionTotal)
case "InnerUnionRows":
statFn(CounterQueryInnerUnionRowsTotal)
case "Xor":
statFn(CounterQueryXorTotal)
case "Not":
statFn(CounterQueryNotTotal)
case "Shift":
statFn(CounterQueryShiftTotal)
case "All":
statFn(CounterQueryAllTotal)
default:
statFn(CounterQueryBitmapTotal)
}
}
// Execute calls in bulk on each remote node and merge.

4
field.go
View file

@ -17,7 +17,6 @@ import (
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/testhook"
"github.com/molecula/featurebase/v3/tracing"
"github.com/pkg/errors"
@ -83,7 +82,6 @@ type Field struct {
viewMap map[string]*view
broadcaster broadcaster
Stats stats.StatsClient
serializer Serializer
// Field options.
@ -395,7 +393,6 @@ func newField(holder *Holder, path, index, name string, opts FieldOption) (*Fiel
viewMap: make(map[string]*view),
broadcaster: NopBroadcaster,
Stats: stats.NopStatsClient,
serializer: NopSerializer,
options: applyDefaultOptions(&fo),
@ -1164,7 +1161,6 @@ func (f *Field) newView(path, name string) *view {
view := newView(f.holder, path, f.index, f.name, name, f.options)
view.idx = f.idx
view.fld = f
view.stats = f.Stats
view.broadcaster = f.broadcaster
return view
}

19
fragment.go
View file

@ -24,7 +24,6 @@ import (
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/shardwidth"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/testhook"
"github.com/molecula/featurebase/v3/tracing"
"github.com/molecula/featurebase/v3/vprint"
@ -119,8 +118,6 @@ type fragment struct {
// mutexVector is used for mutex field types. It's checked for an
// existing value (to clear) prior to setting a new value.
mutexVector vector
stats stats.StatsClient
}
// newFragment returns a new instance of fragment.
@ -141,8 +138,6 @@ func newFragment(holder *Holder, idx *Index, fld *Field, vw *view, shard uint64)
CacheSize: DefaultCacheSize,
holder: holder,
stats: stats.NopStatsClient,
}
return f
}
@ -404,7 +399,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo
f.cache.Add(rowID, n)
}
f.stats.Count(MetricSetBit, 1, 1.0)
CounterSetBit.Inc()
return changed, nil
}
@ -453,7 +448,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b
f.cache.Add(rowID, n)
}
f.stats.Count(MetricClearBit, 1, 1.0)
CounterClearBit.Inc()
return changed, nil
}
@ -509,7 +504,7 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo
}
}
f.stats.Count("setRow", 1, 1.0)
CounterSetRow.Inc()
return changed, nil
}
@ -1712,23 +1707,23 @@ func (p parallelSlices) Swap(i, j int) {
// operations to the op log.
func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64]struct{}) error {
if len(set) > 0 {
f.stats.Count(MetricImportingN, int64(len(set)), 1)
CounterImportingN.Add(float64(len(set)))
// TODO benchmark Add/RemoveN behavior with sorted/unsorted positions
changedN, err := tx.Add(f.index(), f.field(), f.view(), f.shard, set...)
if err != nil {
return errors.Wrap(err, "adding positions")
}
f.stats.Count(MetricImportedN, int64(changedN), 1)
CounterImportedN.Add(float64(changedN))
}
if len(clear) > 0 {
f.stats.Count(MetricClearingN, int64(len(clear)), 1)
CounterClearingingN.Add(float64(len(clear)))
changedN, err := tx.Remove(f.index(), f.field(), f.view(), f.shard, clear...)
if err != nil {
return errors.Wrap(err, "clearing positions")
}
f.stats.Count(MetricClearedN, int64(changedN), 1)
CounterClearedN.Add(float64(changedN))
}
return f.updateCaching(tx, rowSet)
}

2
go.mod
View file

@ -8,7 +8,7 @@ replace robpike.io/ivy => github.com/tgruben/ivy v0.0.0-20221107170120-634b546dc
require (
github.com/CAFxX/gcnotifier v0.0.0-20220409005548-0153238b886a
github.com/DataDog/datadog-go v4.8.3+incompatible
github.com/DataDog/datadog-go v4.8.3+incompatible // indirect
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2

15
holder.go
View file

@ -17,7 +17,6 @@ import (
"github.com/molecula/featurebase/v3/logger"
rbfcfg "github.com/molecula/featurebase/v3/rbf/cfg"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/storage"
"github.com/molecula/featurebase/v3/testhook"
"github.com/molecula/featurebase/v3/vprint"
@ -79,9 +78,6 @@ type Holder struct {
wg sync.WaitGroup
closing chan struct{}
// Stats
Stats stats.StatsClient
// Data directory path.
path string
@ -256,7 +252,6 @@ type HolderConfig struct {
Schemator disco.Schemator
Sharder disco.Sharder
CacheFlushInterval time.Duration
StatsClient stats.StatsClient
Logger logger.Logger
StorageConfig *storage.Config
@ -281,7 +276,6 @@ func DefaultHolderConfig() *HolderConfig {
Schemator: disco.NewInMemSchemator(),
Sharder: disco.InMemSharder,
CacheFlushInterval: defaultCacheFlushInterval,
StatsClient: stats.NopStatsClient,
Logger: logger.NopLogger,
StorageConfig: storage.NewDefaultConfig(),
RBFConfig: rbfcfg.NewDefaultConfig(),
@ -323,7 +317,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
broadcaster: NopBroadcaster,
partitionN: cfg.PartitionN,
Stats: cfg.StatsClient,
cacheFlushInterval: cfg.CacheFlushInterval,
OpenTranslateStore: cfg.OpenTranslateStore,
OpenTranslateReader: cfg.OpenTranslateReader,
@ -526,8 +519,6 @@ func (h *Holder) Open() error {
// Check if deletion was in progress when server was shutdown
h.processDeleteInflight()
h.Stats.Open()
h.opened.Close()
_ = testhook.Opened(h.Auditor, h, nil)
@ -628,8 +619,6 @@ func (h *Holder) Close() error {
fmt.Printf("%v\n", globalCallStats.report())
}
h.Stats.Close()
// Notify goroutines of closing and wait for completion.
close(h.closing)
h.wg.Wait()
@ -1156,7 +1145,6 @@ func (h *Holder) newIndex(path, name string) (*Index, error) {
if err != nil {
return nil, err
}
index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name()))
index.broadcaster = h.broadcaster
index.serializer = h.serializer
index.OpenTranslateStore = h.OpenTranslateStore
@ -1329,9 +1317,6 @@ type holderSyncer struct {
syncers errgroup.Group
// Stats
Stats stats.StatsClient
// Signals that the sync should stop.
Closing <-chan struct{}
}

110
http_handler.go
View file

@ -8,7 +8,6 @@ import (
"encoding/gob"
"encoding/hex"
"encoding/json"
"expvar"
"fmt"
"io"
"math"
@ -42,8 +41,11 @@ import (
"github.com/molecula/featurebase/v3/rbf"
"github.com/molecula/featurebase/v3/sql3/planner/types"
"github.com/molecula/featurebase/v3/storage"
"github.com/molecula/featurebase/v3/wireprotocol"
"github.com/molecula/featurebase/v3/tracing"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
@ -401,8 +403,7 @@ func (h *Handler) collectStats(next http.Handler) http.Handler {
next.ServeHTTP(w, r)
dur := time.Since(t)
statsTags := make([]string, 0, 5)
isSlow := "false"
longQueryTime := h.api.LongQueryTime()
if longQueryTime > 0 && dur > longQueryTime {
queryRequest := r.Context().Value(contextKeyQueryRequest)
@ -413,31 +414,27 @@ func (h *Handler) collectStats(next http.Handler) http.Handler {
}
h.logger.Printf("HTTP query duration %v exceeds %v: %s %s %s", dur, longQueryTime, r.Method, r.URL.String(), queryString)
statsTags = append(statsTags, "slow:true")
} else {
statsTags = append(statsTags, "slow:false")
isSlow = "true"
}
where := ""
pathParts := strings.Split(r.URL.Path, "/")
if externalPrefixFlag[pathParts[1]] {
statsTags = append(statsTags, "where:external")
where = "external"
} else {
statsTags = append(statsTags, "where:internal")
where = "internal"
}
statsTags = append(statsTags, "useragent:"+r.UserAgent())
path, err := mux.CurrentRoute(r).GetPathTemplate()
if err == nil {
statsTags = append(statsTags, "path:"+path)
}
statsTags = append(statsTags, "method:"+r.Method)
stats := h.api.StatsWithTags(statsTags)
if stats != nil {
stats.Timing(MetricHTTPRequest, dur, 0.1)
if err != nil {
path = ""
}
SummaryHttpRequests.With(prometheus.Labels{
"method": r.Method,
"path": path,
"slow": isSlow,
"useragent": r.UserAgent(),
"where": where,
}).Observe(dur.Seconds())
})
}
@ -505,7 +502,6 @@ func newRouter(handler *Handler) http.Handler {
// TODO: figure out how to protect these if needed
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
router.PathPrefix("/debug/fgprof").Handler(fgprof.Handler()).Methods("GET")
router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
router.Handle("/metrics", promhttp.Handler())
router.HandleFunc("/metrics.json", handler.chkAuthZ(handler.handleGetMetricsJSON, authz.Admin)).Methods("GET").Name("GetMetricsJSON")
@ -551,6 +547,8 @@ func newRouter(handler *Handler) http.Handler {
if handler.sqlEnabled {
router.HandleFunc("/sql", handler.chkAuthZ(handler.handlePostSQL, authz.Admin)).Methods("POST").Name("PostSQL")
}
// internal endpoint
router.HandleFunc("/sql", handler.chkAuthZ(handler.handlePostSQLPlanOperator, authz.Admin)).Headers("X-FeatureBase-Plan-Operator", "").Methods("POST").Name("PostSQLPlanOperator")
router.HandleFunc("/query-history", handler.chkAuthZ(handler.handleGetPastQueries, authz.Admin)).Methods("GET").Name("GetPastQueries")
router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion")
@ -1392,10 +1390,70 @@ func (h *Handler) writeBadRequest(w http.ResponseWriter, r *http.Request, err er
}
}
// handlePostSQL handles /sql requests.
func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
includePlan := false
// handlePostSQLOperator handles an internal sql3 plan operator execution request
// these requests come from other nodes in the cluster
// handlePostSQLOperator will 'rehydrate' a plan operator and return data in the
// featurebase wire format for effciency
// we do not track these requests as user requests
// TODO(pok) - thus is there anything we need here to align with how we do this for other nodes
func (h *Handler) handlePostSQLPlanOperator(w http.ResponseWriter, r *http.Request) {
writeError := func(err error) {
if err != nil {
w.Write(wireprotocol.WriteError(err))
}
}
// always finish with a done message
defer w.Write(wireprotocol.WriteDone())
ctx := r.Context()
rootOperator, err := h.api.RehydratePlanOperator(ctx, r.Body)
if err != nil {
writeError(err)
return
}
// get a query iterator.
iter, err := rootOperator.Iterator(ctx, nil)
if err != nil {
writeError(err)
return
}
// read schema & write to response.
columns := rootOperator.Schema()
b, err := wireprotocol.WriteSchema(columns)
if err != nil {
writeError(err)
return
}
w.Write(b)
var rowErr error
var currentRow types.Row
var nextErr error
for currentRow, nextErr = iter.Next(ctx); nextErr == nil; currentRow, nextErr = iter.Next(ctx) {
b, err := wireprotocol.WriteRow(currentRow, columns)
if err != nil {
rowErr = err
break
}
w.Write(b)
}
if nextErr != nil && nextErr != types.ErrNoMoreRows {
rowErr = nextErr
}
writeError(rowErr)
}
// handlePostSQL handles /sql requests
// supports a ?plan=true|false parameter to send back the plan in the
// query response
func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
includePlan := false
includePlanValue := r.URL.Query().Get("plan")
if len(includePlanValue) > 0 {
var err error
@ -1406,6 +1464,7 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
}
}
// get the body
b, err := io.ReadAll(r.Body)
if err != nil {
h.writeBadRequest(w, r, err)
@ -1419,6 +1478,9 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
// put the requestId in the context
ctx := fbcontext.WithRequestID(r.Context(), requestID.String())
// update the counter for requests
PerfCounterSQLRequestSec.Add(1)
// Write response back to client.
w.Header().Set("Content-Type", "application/json")

71
idk/ingest.go
View file

@ -34,9 +34,7 @@ import (
"github.com/molecula/featurebase/v3/idk/mds"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/prometheus"
proto "github.com/molecula/featurebase/v3/proto"
"github.com/molecula/featurebase/v3/stats"
"github.com/pkg/errors"
prom "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
@ -55,7 +53,6 @@ const (
)
// TODO Jaeger
// TODO Prometheus
// Main holds all config for general ingest
type Main struct {
@ -132,7 +129,6 @@ type Main struct {
newNexter func(c int) (IDAllocator, error)
ra RangeAllocator
stats stats.StatsClient
metricsServer *http.Server
log logger.Logger
@ -218,7 +214,7 @@ func NewMain() *Main {
Concurrency: 1,
CacheLength: 64,
PackBools: "bools",
Namespace: "ingester",
Namespace: "ingester", // this is now ignored and hardcoded in metrics.go
IDAllocKeyPrefix: "ingest",
UseShardTransactionalEndpoint: os.Getenv("IDK_DEFAULT_SHARD_TRANSACTIONAL") != "",
@ -228,8 +224,6 @@ func NewMain() *Main {
SchemaManager: NopSchemaManager,
stats: stats.NopStatsClient,
log: logger.NewStandardLogger(os.Stderr),
}
}
@ -457,7 +451,6 @@ initialFetch:
if v, ok := source.(Metadata); ok {
m.log.Printf("new schema - subject: %#v; version: %d; schema: %#v",
v.SchemaSubject(), v.SchemaVersion(), v.SchemaSchema())
// m.log.Printf("new schema: %#v", v.SchemaMetadata())
} else {
m.log.Printf("new schema: %#v", schema)
}
@ -465,7 +458,7 @@ initialFetch:
if err != nil {
return errors.Wrap(err, "batchFromSchema")
}
m.stats.Count(MetricIngesterSchemaChanges, 1, 1)
CounterIngesterSchemaChanges.Inc()
csvSlice = make([]string, len(schema))
if m.csvWriter != nil {
for i := range schema {
@ -571,7 +564,7 @@ initialFetch:
// skip bad rows only
if !rowHasError {
err = batch.Add(*row)
m.stats.Count(MetricIngesterRowsAdded, 1, 1)
CounterIngesterRowsAdded.Inc()
}
if err == pilosabatch.ErrBatchNowFull || err == pilosabatch.ErrBatchNowStale {
@ -958,7 +951,7 @@ func (m *Main) commitRecord(ctx context.Context, rec Record, limitCounter *msgCo
return errors.Wrap(err, "committing")
}
limitCounter.Increment(numRecords)
m.stats.Count(MetricCommittedRecords, int64(numRecords), 1)
CounterCommittedRecords.Add(float64(numRecords))
return nil
}
@ -972,7 +965,7 @@ func (m *Main) NewLookupClient() (*PostgresClient, error) {
func (m *Main) setupClient() (*tls.Config, error) {
var tlsConfig *tls.Config
var err error
var opts = []pilosaclient.ClientOption{pilosaclient.OptClientStatsClient(m.stats)}
var opts = []pilosaclient.ClientOption{}
if m.TLS.CertificatePath != "" {
tlsConfig, err = GetTLSConfig(&m.TLS, m.Log())
if err != nil {
@ -1037,34 +1030,24 @@ func (m *Main) setupClient() (*tls.Config, error) {
}
func (m *Main) setupStats() error {
if m.Stats != "" {
opts := []prometheus.ClientOption{prometheus.OptClientNamespace(m.Namespace)}
m.stats, _ = prometheus.NewPrometheusClient(opts...) // ignore error that must be nil
mux := http.NewServeMux()
// reg := prom.NewRegistry() // TODO switch to this once pilosa PrometheusClient is fixed and doesn't use the global registry internally.
// also change prom.DefaultGatherer to be "reg" at that time
reg := prom.DefaultRegisterer
promHandler := promhttp.InstrumentMetricHandler(reg, promhttp.HandlerFor(prom.DefaultGatherer, promhttp.HandlerOpts{}))
mux.Handle("/metrics", promHandler)
mux.Handle("/metrics.json", metricsJSONHandler{metricsURI: "http://" + m.Stats + "/metrics"})
m.metricsServer = &http.Server{Addr: m.Stats, Handler: mux}
ln, err := net.Listen("tcp", m.Stats)
if err != nil {
return errors.Wrapf(err, "listen for metrics on '%s'", m.Stats)
}
go func() {
m.log.Printf("Serving Prometheus metrics with namespace \"%s\" at %v/metrics\n", m.Namespace, m.Stats)
err = m.metricsServer.Serve(ln)
if err != http.ErrServerClosed {
m.log.Printf("serve metrics on '%s': %v", m.Stats, err)
}
}()
mux := http.NewServeMux()
promHandler := promhttp.InstrumentMetricHandler(prom.DefaultRegisterer, promhttp.HandlerFor(prom.DefaultGatherer, promhttp.HandlerOpts{}))
mux.Handle("/metrics", promHandler)
mux.Handle("/metrics.json", metricsJSONHandler{metricsURI: "http://" + m.Stats + "/metrics"})
m.metricsServer = &http.Server{Addr: m.Stats, Handler: mux}
ln, err := net.Listen("tcp", m.Stats)
if err != nil {
return errors.Wrapf(err, "listen for metrics on '%s'", m.Stats)
}
return nil
go func() {
m.log.Printf("Serving Prometheus metrics with namespace \"%s\" at %v/metrics\n", m.Namespace, m.Stats)
err = m.metricsServer.Serve(ln)
if err != http.ErrServerClosed {
m.log.Printf("serve metrics on '%s': %v", m.Stats, err)
}
}()
return nil
}
type metricsJSONHandler struct {
@ -1218,7 +1201,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error {
if err != nil {
return errors.Wrap(err, "clearing bools")
}
m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:packed-bool"})
CounterDeleterRowsAdded.With(prom.Labels{"type": "packed-bool"}).Inc()
continue
} else {
fieldName = directive
@ -1267,7 +1250,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error {
if err != nil {
return errors.Wrap(err, "clearing set")
}
m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:set"})
CounterDeleterRowsAdded.With(prom.Labels{"type": "set"}).Inc()
case pilosaclient.FieldTypeMutex:
if val == "" {
continue
@ -1278,7 +1261,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error {
if err != nil {
return errors.Wrap(err, "clearing mutex")
}
m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:mutex"})
CounterDeleterRowsAdded.With(prom.Labels{"type": "mutex"}).Inc()
case pilosaclient.FieldTypeBool:
_, err := client.Query(index.BatchQuery(
field.Clear(0, recordID),
@ -1287,19 +1270,19 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error {
if err != nil {
return errors.Wrap(err, "clearing bool")
}
m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:bool"})
CounterDeleterRowsAdded.With(prom.Labels{"type": "bool"}).Inc()
case pilosaclient.FieldTypeInt:
_, err := client.Query(field.Clear(0, recordID))
if err != nil {
return errors.Wrap(err, "clearing int")
}
m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:int"})
CounterDeleterRowsAdded.With(prom.Labels{"type": "int"}).Inc()
case pilosaclient.FieldTypeDecimal:
_, err := client.Query(field.Clear(0, recordID))
if err != nil {
return errors.Wrap(err, "clearing decimal")
}
m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:decimal"})
CounterDeleterRowsAdded.With(prom.Labels{"type": "decimal"}).Inc()
case pilosaclient.FieldTypeTime:
return errors.Errorf("deletion on time fields unimplemented")
default:

2
idk/ingest_test.go
View file

@ -525,7 +525,7 @@ func TestIngesterServesPrometheusEndpoint(t *testing.T) {
if err != nil {
t.Errorf("read error: %v", err)
}
if strings.Contains(string(contents), MetricIngesterRowsAdded) {
if !strings.Contains(string(contents), MetricIngesterRowsAdded) {
t.Errorf("metric name missing: %v", MetricIngesterRowsAdded)
}
close(records)

2
idk/mds/importer.go
View file

@ -248,8 +248,6 @@ func (m *importer) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field
return fbClient.DoImport(string(qtbl.Key()), shard, path, data)
}
func (m *importer) StatsTiming(name string, value time.Duration, rate float64) {}
// getQtbl takes a table (TableKey) and sets the local m.qtbl value. When we
// originally set up this type, it was only used by IDK, and the table was known
// at the beginning of the process, so it could be set on this import. But

44
idk/metrics.go
View file

@ -1,8 +1,52 @@
package idk
import "github.com/prometheus/client_golang/prometheus"
const (
MetricDeleterRowsAdded = "deleter_rows_added_total"
MetricIngesterRowsAdded = "ingester_rows_added_total"
MetricIngesterSchemaChanges = "ingester_schema_changes_total"
MetricCommittedRecords = "committed_records"
)
var CounterIngesterSchemaChanges = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: "ingester",
Name: MetricIngesterSchemaChanges,
Help: "TODO",
},
)
var CounterIngesterRowsAdded = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: "ingester",
Name: MetricIngesterRowsAdded,
Help: "TODO",
},
)
var CounterCommittedRecords = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: "ingester",
Name: MetricCommittedRecords,
Help: "TODO",
},
)
var CounterDeleterRowsAdded = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "ingester",
Name: MetricDeleterRowsAdded,
Help: "TODO",
},
[]string{
"type",
},
)
func init() {
prometheus.MustRegister(CounterIngesterSchemaChanges)
prometheus.MustRegister(CounterIngesterRowsAdded)
prometheus.MustRegister(CounterCommittedRecords)
prometheus.MustRegister(CounterDeleterRowsAdded)
}

2
importer.go
View file

@ -20,8 +20,6 @@ type Importer interface {
EncodeImportValues(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, vals []int64, ids []uint64, clear bool) (path string, data []byte, err error)
EncodeImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, vals, ids []uint64, clear bool) (path string, data []byte, err error)
DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, path string, data []byte) error
StatsTiming(name string, value time.Duration, rate float64)
}
// Ensure type implements interface.

7
index.go
View file

@ -16,9 +16,9 @@ import (
"github.com/molecula/featurebase/v3/disco"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/testhook"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"golang.org/x/sync/errgroup"
)
@ -43,7 +43,6 @@ type Index struct {
broadcaster broadcaster
serializer Serializer
Stats stats.StatsClient
// Passed to field for foreign-index lookup.
holder *Holder
@ -82,7 +81,6 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
fields: make(map[string]*Field),
broadcaster: NopBroadcaster,
Stats: stats.NopStatsClient,
holder: holder,
trackExistence: true,
@ -510,7 +508,7 @@ func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap {
b.UnionInPlace(f.AvailableShards(localOnly))
}
i.Stats.Gauge(MetricMaxShard, float64(b.Max()), 1.0)
GaugeIndexMaxShard.With(prometheus.Labels{"index": i.name}).Set(float64(b.Max()))
return b
}
@ -923,7 +921,6 @@ func (i *Index) newField(path, name string) (*Field, error) {
return nil, err
}
f.idx = i
f.Stats = i.Stats
f.broadcaster = i.broadcaster
f.serializer = i.serializer
f.OpenTranslateStore = i.OpenTranslateStore

996
metrics.go
View file

File diff suppressed because it is too large Load diff

203
performancecounters.go Normal file
View file

@ -0,0 +1,203 @@
package pilosa
import (
"sync/atomic"
"github.com/prometheus/client_golang/prometheus"
)
// PerformanceCounter holds data about a performance counter for external consumers
type PerformanceCounter struct {
NameSpace string
SubSystem string
CounterName string
Help string
Value int64
CounterType int64
}
// constants for the counter types
const (
// raw - for when you just want a count of something
CTR_TYPE_RAW = 0
// per second - for when you accumulate counts of things
// a consumer would sample this at intervals to arrive at a delta
// then divide by the time in seconds between the samples to get a
// per-second value
CTR_TYPE_PER_SECOND = 1
// ratio - for when you accumulate a count of something that you
// want to use as a numerator in a ratio calculation
// e.g. 'cache hits' could be a counter of this type and you could
// divide it by a 'cache lookups' counter to get the hit ratio (see below)
CTR_TYPE_RATIO = 2
// ratio base - for when you accumulate a count of something that you
// want to use as a denominator in a ratio calculation
// e.g. 'cache lookups' could be a counter of this type and you could
// use it as the denominator in a division with a 'cache hits' counter
// as the numerator to get the hit ratio
CTR_TYPE_RATIO_BASE = 3
)
type PerformanceCounters struct {
counterValues [5]perfCtrWrapper
}
type perfCtr struct {
nameSpace string
subSystem string
name string
help string
value int64
counterType int64
}
func (p *perfCtr) Add(increment int64) {
atomic.AddInt64(&p.value, increment)
}
type perfCtrWrapper struct {
ctr *perfCtr
fun prometheus.CounterFunc
}
var PerfCounterSQLRequestSec = perfCtr{
nameSpace: "pilosa",
subSystem: "sql_statistics",
name: "sql_requests_sec",
help: "TODO",
value: 0,
counterType: CTR_TYPE_PER_SECOND,
}
var PerfCounterSQLInsertsSec = perfCtr{
nameSpace: "pilosa",
subSystem: "sql_statistics",
name: "sql_inserts_sec",
help: "TODO",
value: 0,
counterType: CTR_TYPE_PER_SECOND,
}
var PerfCounterSQLBulkInsertsSec = perfCtr{
nameSpace: "pilosa",
subSystem: "sql_statistics",
name: "sql_bulk_inserts_sec",
help: "TODO",
value: 0,
counterType: CTR_TYPE_PER_SECOND,
}
var PerfCounterSQLBulkInsertBatchesSec = perfCtr{
nameSpace: "pilosa",
subSystem: "sql_statistics",
name: "sql_bulk_insert_batches_sec",
help: "TODO",
value: 0,
counterType: CTR_TYPE_PER_SECOND,
}
var PerfCounterSQLDeletesSec = perfCtr{
nameSpace: "pilosa",
subSystem: "sql_statistics",
name: "sql_deletes_sec",
help: "TODO",
value: 0,
counterType: CTR_TYPE_PER_SECOND,
}
var PerfCounters *PerformanceCounters = newPerformanceCounters()
func newPerformanceCounters() *PerformanceCounters {
ctrs := &PerformanceCounters{
counterValues: [5]perfCtrWrapper{
{
&PerfCounterSQLRequestSec,
prometheus.NewCounterFunc(
prometheus.CounterOpts{
Namespace: PerfCounterSQLRequestSec.nameSpace,
Subsystem: PerfCounterSQLRequestSec.subSystem,
Name: PerfCounterSQLRequestSec.name,
Help: PerfCounterSQLRequestSec.help,
},
func() float64 {
return float64(atomic.LoadInt64(&PerfCounterSQLRequestSec.value))
}),
},
{
&PerfCounterSQLInsertsSec,
prometheus.NewCounterFunc(
prometheus.CounterOpts{
Namespace: PerfCounterSQLInsertsSec.nameSpace,
Subsystem: PerfCounterSQLInsertsSec.subSystem,
Name: PerfCounterSQLInsertsSec.name,
Help: PerfCounterSQLInsertsSec.help,
},
func() float64 {
return float64(atomic.LoadInt64(&PerfCounterSQLInsertsSec.value))
}),
},
{
&PerfCounterSQLBulkInsertsSec,
prometheus.NewCounterFunc(
prometheus.CounterOpts{
Namespace: PerfCounterSQLBulkInsertsSec.nameSpace,
Subsystem: PerfCounterSQLBulkInsertsSec.subSystem,
Name: PerfCounterSQLBulkInsertsSec.name,
Help: PerfCounterSQLBulkInsertsSec.help,
},
func() float64 {
return float64(atomic.LoadInt64(&PerfCounterSQLBulkInsertsSec.value))
}),
},
{
&PerfCounterSQLBulkInsertBatchesSec,
prometheus.NewCounterFunc(
prometheus.CounterOpts{
Namespace: PerfCounterSQLBulkInsertBatchesSec.nameSpace,
Subsystem: PerfCounterSQLBulkInsertBatchesSec.subSystem,
Name: PerfCounterSQLBulkInsertBatchesSec.name,
Help: PerfCounterSQLBulkInsertBatchesSec.help,
},
func() float64 {
return float64(atomic.LoadInt64(&PerfCounterSQLBulkInsertBatchesSec.value))
}),
},
{
&PerfCounterSQLDeletesSec,
prometheus.NewCounterFunc(
prometheus.CounterOpts{
Namespace: PerfCounterSQLDeletesSec.nameSpace,
Subsystem: PerfCounterSQLDeletesSec.subSystem,
Name: PerfCounterSQLDeletesSec.name,
Help: PerfCounterSQLDeletesSec.help,
},
func() float64 {
return float64(atomic.LoadInt64(&PerfCounterSQLDeletesSec.value))
}),
},
},
}
for _, w := range ctrs.counterValues {
prometheus.MustRegister(w.fun)
}
return ctrs
}
// list all the counters
// we can just read here without locking because if the counters get changed
// midway thru the loop, absent evidence to the contrary, the world will not end
func (p *PerformanceCounters) ListCounters() ([]PerformanceCounter, error) {
result := make([]PerformanceCounter, len(p.counterValues))
for i, c := range p.counterValues {
result[i] = PerformanceCounter{
NameSpace: c.ctr.nameSpace,
SubSystem: c.ctr.subSystem,
CounterName: c.ctr.name,
Value: c.ctr.value,
CounterType: c.ctr.counterType,
}
}
return result, nil
}

307
prometheus/prometheus.go
View file

@ -1,307 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package prometheus
import (
"sort"
"strings"
"sync"
"time"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/stats"
"github.com/prometheus/client_golang/prometheus"
)
const (
// namespace is prepended to each metric event name with "_"
defaultNamespace = "general"
)
// Ensure client implements interface.
var _ stats.StatsClient = &prometheusClient{}
// Module-level mutex to avoid copying in WithTags()
var mu sync.Mutex
// prometheusClient represents a Prometheus implementation of pilosa.statsClient.
type prometheusClient struct {
tags []string
logger logger.Logger
counters map[string]prometheus.Counter
counterVecs map[string]*prometheus.CounterVec
gauges map[string]prometheus.Gauge
gaugeVecs map[string]*prometheus.GaugeVec
observers map[string]prometheus.Observer
summaryVecs map[string]*prometheus.SummaryVec
namespace string
}
// ClientOption is a functional option type for prometheusClient
type ClientOption func(c *prometheusClient)
// OptClientPrefix is a functional option on prometheusClient used to set the namespace
func OptClientNamespace(namespace string) ClientOption {
return func(c *prometheusClient) {
c.namespace = namespace
}
}
// NewPrometheusClient returns a new instance of StatsClient.
func NewPrometheusClient(opts ...ClientOption) (*prometheusClient, error) {
client := &prometheusClient{
logger: logger.NopLogger,
counters: make(map[string]prometheus.Counter),
counterVecs: make(map[string]*prometheus.CounterVec),
gauges: make(map[string]prometheus.Gauge),
gaugeVecs: make(map[string]*prometheus.GaugeVec),
observers: make(map[string]prometheus.Observer),
summaryVecs: make(map[string]*prometheus.SummaryVec),
namespace: defaultNamespace,
}
for _, opt := range opts {
opt(client)
}
return client, nil
}
// Open no-op to satisfy interface
func (c *prometheusClient) Open() {}
// Close no-op to satisfy interface
func (c *prometheusClient) Close() error {
return nil
}
// Tags returns a sorted list of tags on the client.
func (c *prometheusClient) Tags() []string {
return c.tags
}
// labels returns an instance of prometheus.Labels with the value of the set tags.
func (c *prometheusClient) labels() prometheus.Labels {
return tagsToLabels(c.tags, c.logger)
}
// WithTags returns a new client with additional tags appended.
func (c *prometheusClient) WithTags(tags ...string) stats.StatsClient {
return &prometheusClient{
tags: unionStringSlice(c.tags, tags),
logger: c.logger,
counters: c.counters,
counterVecs: c.counterVecs,
gauges: c.gauges,
gaugeVecs: c.gaugeVecs,
observers: c.observers,
summaryVecs: c.summaryVecs,
namespace: c.namespace,
}
}
// Count tracks the number of times something occurs per second.
func (c *prometheusClient) Count(name string, value int64, rate float64) {
mu.Lock()
defer mu.Unlock()
var counter prometheus.Counter
var ok bool
name = strings.Replace(name, ".", "_", -1)
labels := c.labels()
opts := prometheus.CounterOpts{
Namespace: c.namespace,
Name: name,
}
if len(labels) == 0 {
counter, ok = c.counters[name]
if !ok {
counter = prometheus.NewCounter(opts)
c.counters[name] = counter
prometheus.MustRegister(counter)
}
} else {
var counterVec *prometheus.CounterVec
counterVec, ok = c.counterVecs[name]
if !ok {
counterVec = prometheus.NewCounterVec(
opts,
labelKeys(labels),
)
c.counterVecs[name] = counterVec
prometheus.MustRegister(counterVec)
}
var err error
counter, err = counterVec.GetMetricWith(labels)
if err != nil {
c.logger.Errorf("counterVec.GetMetricWith error: %s", err)
}
}
if value == 1 {
counter.Inc()
} else {
counter.Add(float64(value))
}
}
// CountWithCustomTags tracks the number of times something occurs per second with custom tags.
func (c *prometheusClient) CountWithCustomTags(name string, value int64, rate float64, t []string) {
c.WithTags(append(c.tags, t...)...).Count(name, value, rate)
}
// Gauge sets the value of a metric.
func (c *prometheusClient) Gauge(name string, value float64, rate float64) {
mu.Lock()
defer mu.Unlock()
var gauge prometheus.Gauge
var ok bool
name = strings.Replace(name, ".", "_", -1)
labels := c.labels()
opts := prometheus.GaugeOpts{
Namespace: c.namespace,
Name: name,
}
if len(labels) == 0 {
gauge, ok = c.gauges[name]
if !ok {
gauge = prometheus.NewGauge(opts)
c.gauges[name] = gauge
prometheus.MustRegister(gauge)
}
} else {
var gaugeVec *prometheus.GaugeVec
gaugeVec, ok = c.gaugeVecs[name]
if !ok {
gaugeVec = prometheus.NewGaugeVec(
opts,
labelKeys(labels),
)
c.gaugeVecs[name] = gaugeVec
prometheus.MustRegister(gaugeVec)
}
var err error
gauge, err = gaugeVec.GetMetricWith(labels)
if err != nil {
c.logger.Errorf("gaugeVec.GetMetricWith error: %s", err)
return
}
}
gauge.Set(float64(value))
}
// Histogram tracks statistical distribution of a metric.
func (c *prometheusClient) Histogram(name string, value float64, rate float64) {
mu.Lock()
defer mu.Unlock()
var observer prometheus.Observer
var ok bool
name = strings.Replace(name, ".", "_", -1)
labels := c.labels()
opts := prometheus.SummaryOpts{
Namespace: c.namespace,
Name: name,
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}
if len(labels) == 0 {
observer, ok = c.observers[name]
if !ok {
summary := prometheus.NewSummary(opts)
observer = summary
c.observers[name] = observer
prometheus.MustRegister(summary)
}
} else {
var summaryVec *prometheus.SummaryVec
summaryVec, ok = c.summaryVecs[name]
if !ok {
summaryVec = prometheus.NewSummaryVec(
opts,
labelKeys(labels),
)
c.summaryVecs[name] = summaryVec
prometheus.MustRegister(summaryVec)
}
var err error
observer, err = summaryVec.GetMetricWith(labels)
if err != nil {
c.logger.Errorf("summaryVec.GetMetricWith error: %s", err)
return
}
}
observer.Observe(value)
}
// Set tracks number of unique elements.
func (c *prometheusClient) Set(name string, value string, rate float64) {
c.logger.Infof("prometheusClient.Set unimplemented: %s=%s", name, value)
}
// Timing tracks timing information for a metric.
func (c *prometheusClient) Timing(name string, value time.Duration, rate float64) {
c.Histogram(name, value.Seconds(), rate)
}
// SetLogger sets the logger for client.
func (c *prometheusClient) SetLogger(logger logger.Logger) {
c.logger = logger
}
// unionStringSlice returns a sorted set of tags which combine a & b.
func unionStringSlice(a, b []string) []string {
// Sort both sets first.
sort.Strings(a)
sort.Strings(b)
// Find size of largest slice.
n := len(a)
if len(b) > n {
n = len(b)
}
// Exit if both sets are empty.
if n == 0 {
return nil
}
// Iterate over both in order and merge.
other := make([]string, 0, n)
for len(a) > 0 || len(b) > 0 {
if len(a) == 0 {
other, b = append(other, b[0]), b[1:]
} else if len(b) == 0 {
other, a = append(other, a[0]), a[1:]
} else if a[0] < b[0] {
other, a = append(other, a[0]), a[1:]
} else if b[0] < a[0] {
other, b = append(other, b[0]), b[1:]
} else {
other, a, b = append(other, a[0]), a[1:], b[1:]
}
}
return other
}
func tagsToLabels(tags []string, logger logger.Logger) (labels prometheus.Labels) {
labels = make(prometheus.Labels)
for _, tag := range tags {
tagParts := strings.SplitAfterN(tag, ":", 2)
if len(tagParts) != 2 {
// only process tags in "key:value" form
logger.Errorf("invalid Prometheus label: %v\n", tag)
continue
}
labels[tagParts[0][0:len(tagParts[0])-1]] = tagParts[1]
}
return labels
}
func labelKeys(labels prometheus.Labels) (keys []string) {
keys = make([]string, len(labels))
i := 0
for k := range labels {
keys[i] = k
i++
}
return keys
}

50
prometheus/prometheus_test.go
View file

@ -2,62 +2,32 @@
package prometheus_test
import (
"reflect"
"testing"
"time"
pilosaPrometheus "github.com/molecula/featurebase/v3/prometheus"
"github.com/molecula/featurebase/v3/test"
"github.com/prometheus/client_golang/prometheus"
io_prometheus_client "github.com/prometheus/client_model/go"
)
func TestPrometheusClient_WithTags(t *testing.T) {
// Create a new client.
c, err := pilosaPrometheus.NewPrometheusClient()
if err != nil {
t.Fatal(err)
}
defer c.Close()
// Create a new client with additional tags.
c1 := c.WithTags("foo", "bar")
if tags := c1.Tags(); !reflect.DeepEqual(tags, []string{"bar", "foo"}) {
t.Fatalf("unexpected tags: %+v", tags)
}
// Create a new client from the clone with more tags.
c2 := c1.WithTags("bar", "baz")
if tags := c2.Tags(); !reflect.DeepEqual(tags, []string{"bar", "baz", "foo"}) {
t.Fatalf("unexpected tags: %+v", tags)
}
}
func TestPrometheusClient_Methods(t *testing.T) {
// Create a new client.
c, err := pilosaPrometheus.NewPrometheusClient(
pilosaPrometheus.OptClientNamespace("testns"),
)
if err != nil {
t.Fatal(err)
}
c := test.MustRunCluster(t, 1)
defer c.Close()
dur, _ := time.ParseDuration("123us")
c.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"})
c.Count("cc", 1, 1.0)
c.Gauge("gg", 10, 1.0)
c.Histogram("hh", 1, 1.0)
c.Timing("tt", dur, 1.0)
metricFams, err := prometheus.DefaultGatherer.Gather()
if err != nil {
t.Fatal(err)
}
for _, metricName := range []string{"testns_ct", "testns_cc", "testns_gg", "testns_hh", "testns_tt"} {
for _, metricName := range []string{
"pilosa_sql_statistics_sql_bulk_insert_batches_sec",
"pilosa_sql_statistics_sql_bulk_inserts_sec",
"pilosa_sql_statistics_sql_deletes_sec",
"pilosa_sql_statistics_sql_inserts_sec",
"pilosa_sql_statistics_sql_requests_sec",
} {
if metricExists(metricName, metricFams) {
continue
}
t.Fatalf("Metric was not recorded: %s", metricName)
t.Fatalf("metric does not exist: %s", metricName)
}
}

36
server.go
View file

@ -4,6 +4,7 @@ package pilosa
import (
"context"
"fmt"
"io"
"log"
"os"
"os/exec"
@ -25,7 +26,6 @@ import (
"github.com/molecula/featurebase/v3/sql3"
"github.com/molecula/featurebase/v3/sql3/parser"
planner_types "github.com/molecula/featurebase/v3/sql3/planner/types"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/storage"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
@ -245,15 +245,6 @@ func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption {
}
}
// OptServerStatsClient is a functional option on Server
// used to specify the stats client.
func OptServerStatsClient(sc stats.StatsClient) ServerOption {
return func(s *Server) error {
s.holderConfig.StatsClient = sc
return nil
}
}
// OptServerDiagnosticsInterval is a functional option on Server
// used to specify the duration between diagnostic checks.
func OptServerDiagnosticsInterval(dur time.Duration) ServerOption {
@ -541,7 +532,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
return nil, err
}
s.holder = NewHolder(path, s.holderConfig)
s.holder.Stats.SetLogger(s.logger)
cwd, err := os.Getwd()
if err != nil {
return nil, err
@ -557,9 +547,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.cluster.sharder = s.sharder
s.cluster.serverlessStorage = s.serverlessStorage
// Append the NodeID tag to stats.
s.holder.Stats = s.holder.Stats.WithTags(fmt.Sprintf("node_id:%s", s.nodeID))
s.executor.Holder = s.holder
s.holder.executor = s.executor
s.executor.Cluster = s.cluster
@ -657,7 +644,6 @@ func (s *Server) Open() error {
s.syncer.Node = node
s.syncer.Cluster = s.cluster
s.syncer.Closing = s.closing
s.syncer.Stats = s.holder.Stats.WithTags("component:HolderSyncer")
// Start background process listening for translation
// sync resets.
@ -1249,26 +1235,26 @@ func (s *Server) monitorRuntime() {
return
case <-s.gcNotifier.AfterGC():
// GC just ran.
s.holder.Stats.Count(MetricGarbageCollection, 1, 1.0)
CounterGarbageCollection.Inc()
case <-ticker.C:
}
// Record the number of go routines.
s.holder.Stats.Gauge(MetricGoroutines, float64(runtime.NumGoroutine()), 1.0)
GaugeGoroutines.Set(float64(runtime.NumGoroutine()))
openFiles, err := countOpenFiles()
// Open File handles.
if err == nil {
s.holder.Stats.Gauge(MetricOpenFiles, float64(openFiles), 1.0)
GaugeOpenFiles.Set(float64(openFiles))
}
// Runtime memory metrics.
runtime.ReadMemStats(&m)
s.holder.Stats.Gauge(MetricHeapAlloc, float64(m.HeapAlloc), 1.0)
s.holder.Stats.Gauge(MetricHeapInuse, float64(m.HeapInuse), 1.0)
s.holder.Stats.Gauge(MetricStackInuse, float64(m.StackInuse), 1.0)
s.holder.Stats.Gauge(MetricMallocs, float64(m.Mallocs), 1.0)
s.holder.Stats.Gauge(MetricFrees, float64(m.Frees), 1.0)
GaugeHeapAlloc.Set(float64(m.HeapAlloc))
GaugeHeapInUse.Set(float64(m.HeapInuse))
GaugeStackInUse.Set(float64(m.StackInuse))
GaugeMallocs.Set(float64(m.Mallocs))
GaugeFrees.Set(float64(m.Frees))
}
}
@ -1412,6 +1398,10 @@ func (s *Server) CompileExecutionPlan(ctx context.Context, q string) (planner_ty
return s.executionPlannerFn(s.executor, s.executor.client.api, q).CompilePlan(ctx, st)
}
func (s *Server) RehydratePlanOperator(ctx context.Context, reader io.Reader) (planner_types.PlanOperator, error) {
return s.executionPlannerFn(s.executor, s.executor.client.api, "").RehydratePlanOp(ctx, reader)
}
// countOpenFiles on operating systems that support lsof.
func countOpenFiles() (int, error) {
switch runtime.GOOS {

43
server/grpc.go
View file

@ -22,7 +22,6 @@ import (
pb "github.com/molecula/featurebase/v3/proto"
vdsm_pb "github.com/molecula/featurebase/v3/proto/vdsm"
"github.com/molecula/featurebase/v3/sql"
"github.com/molecula/featurebase/v3/stats"
"github.com/pkg/errors"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
@ -40,12 +39,11 @@ type GRPCHandler struct {
perms *authz.GroupPermissions
logger logger.Logger
queryLogger logger.Logger
stats stats.StatsClient
inspectDeprecated sync.Once
}
func NewGRPCHandler(api *pilosa.API) *GRPCHandler {
return &GRPCHandler{api: api, logger: logger.NopLogger, stats: stats.NopStatsClient}
return &GRPCHandler{api: api, logger: logger.NopLogger}
}
func (h *GRPCHandler) WithLogger(logger logger.Logger) *GRPCHandler {
@ -53,11 +51,6 @@ func (h *GRPCHandler) WithLogger(logger logger.Logger) *GRPCHandler {
return h
}
func (h *GRPCHandler) WithStats(stats stats.StatsClient) *GRPCHandler {
h.stats = stats
return h
}
func (h *GRPCHandler) WithPerms(perms *authz.GroupPermissions) *GRPCHandler {
h.perms = perms
return h
@ -139,7 +132,7 @@ func errToStatusError(err error) error {
}
func (h *GRPCHandler) execSQL(ctx context.Context, queryStr string) (pb.ToRowser, error) {
h.stats.Count(pilosa.MetricSqlQueries, 1, 1)
pilosa.CounterSQLQueries.Inc()
return execSQL(ctx, h.api, h.logger, queryStr)
}
@ -339,9 +332,9 @@ func (h *GRPCHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQ
return errToStatusError(err)
}
durFormat := time.Since(t)
h.stats.Timing(pilosa.MetricGRPCStreamQueryDurationSeconds, durQuery, 0.1)
h.stats.Timing(pilosa.MetricGRPCStreamFormatDurationSeconds, durFormat, 0.1)
h.stats.Count(pilosa.MetricPqlQueries, 1, 1)
pilosa.SummaryGRPCStreamQueryDurationSeconds.Observe(durQuery.Seconds())
pilosa.SummaryGRPCStreamFormatDurationSeconds.Observe(durFormat.Seconds())
pilosa.CounterPQLQueries.Inc()
return errToStatusError(nil)
}
@ -405,9 +398,9 @@ func (h *GRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest
return nil, errors.Wrap(err, "sending header")
}
h.stats.Timing(pilosa.MetricGRPCUnaryQueryDurationSeconds, durQuery, 0.1)
h.stats.Timing(pilosa.MetricGRPCUnaryFormatDurationSeconds, durFormat, 0.1)
h.stats.Count(pilosa.MetricPqlQueries, 1, 1)
pilosa.SummaryGRPCUnaryQueryDurationSeconds.Observe(durQuery.Seconds())
pilosa.SummaryGRPCUnaryFormatDurationSeconds.Observe(durFormat.Seconds())
pilosa.CounterPQLQueries.Inc()
return table, errToStatusError(nil)
}
@ -509,11 +502,10 @@ type VDSMGRPCHandler struct {
grpcHandler *GRPCHandler
api *pilosa.API
logger logger.Logger
stats stats.StatsClient
}
func NewVDSMGRPCHandler(grpcHandler *GRPCHandler, api *pilosa.API) *VDSMGRPCHandler {
return &VDSMGRPCHandler{grpcHandler: grpcHandler, api: api, logger: logger.NopLogger, stats: stats.NopStatsClient}
return &VDSMGRPCHandler{grpcHandler: grpcHandler, api: api, logger: logger.NopLogger}
}
func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler {
@ -521,11 +513,6 @@ func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler {
return h
}
func (h *VDSMGRPCHandler) WithStats(stats stats.StatsClient) *VDSMGRPCHandler {
h.stats = stats
return h
}
// GetVDSs returns a single VDS given a name
func (h *VDSMGRPCHandler) GetVDS(ctx context.Context, req *vdsm_pb.GetVDSRequest) (*vdsm_pb.GetVDSResponse, error) {
typedIdOrName := req.GetIdOrName()
@ -1480,7 +1467,6 @@ type grpcServer struct {
logger logger.Logger
queryLogger logger.Logger
stats stats.StatsClient
}
type grpcServerOption func(s *grpcServer) error
@ -1513,13 +1499,6 @@ func OptGRPCServerLogger(logger logger.Logger) grpcServerOption {
}
}
func OptGRPCServerStats(stats stats.StatsClient) grpcServerOption {
return func(s *grpcServer) error {
s.stats = stats
return nil
}
}
func OptGRPCServerAuth(authn *authn.Auth) grpcServerOption {
return func(s *grpcServer) error {
s.auth = authn
@ -1639,7 +1618,7 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) {
// create grpc server
server.grpcServer = grpc.NewServer(gopts...)
grpcHandler := NewGRPCHandler(server.api).WithLogger(server.logger).WithStats(server.stats).WithQueryLogger(server.queryLogger)
grpcHandler := NewGRPCHandler(server.api).WithLogger(server.logger).WithQueryLogger(server.queryLogger)
// add server permissions if we've got 'em
if server.perms != nil {
@ -1647,7 +1626,7 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) {
}
pb.RegisterPilosaServer(server.grpcServer, grpcHandler)
vdsm_pb.RegisterMoleculaServer(server.grpcServer, NewVDSMGRPCHandler(grpcHandler, server.api).WithLogger(server.logger).WithStats(server.stats))
vdsm_pb.RegisterMoleculaServer(server.grpcServer, NewVDSMGRPCHandler(grpcHandler, server.api).WithLogger(server.logger))
// register the server so its services are available to grpc_cli and others
reflection.Register(server.grpcServer)

9
server/handler_test.go
View file

@ -1079,15 +1079,6 @@ func TestHandler_Endpoints(t *testing.T) {
}
})
t.Run("Expvars", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/debug/vars", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
})
t.Run("Recalculate Caches", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil))

28
server/server.go
View file

@ -39,12 +39,9 @@ import (
"github.com/molecula/featurebase/v3/gopsutil"
"github.com/molecula/featurebase/v3/logger"
pnet "github.com/molecula/featurebase/v3/net"
"github.com/molecula/featurebase/v3/prometheus"
"github.com/molecula/featurebase/v3/sql3"
"github.com/molecula/featurebase/v3/sql3/planner"
"github.com/molecula/featurebase/v3/statik"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/statsd"
"github.com/molecula/featurebase/v3/systemlayer"
"github.com/molecula/featurebase/v3/syswrap"
"github.com/molecula/featurebase/v3/testhook"
@ -474,11 +471,6 @@ func (m *Command) setupServer() error {
diagnosticsInterval = defaultDiagnosticsInterval
}
statsClient, err := newStatsClient(m.Config.Metric.Service, m.Config.Metric.Host, m.Config.Namespace())
if err != nil {
return errors.Wrap(err, "new stats client")
}
if m.Config.Listener == nil {
m.ln, err = getListener(*uri, m.tlsConfig)
if err != nil {
@ -581,7 +573,6 @@ func (m *Command) setupServer() error {
pilosa.OptServerQueryLogger(m.queryLogger),
pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),
pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
pilosa.OptServerStatsClient(statsClient),
pilosa.OptServerURI(advertiseURI),
pilosa.OptServerGRPCURI(advertiseGRPCURI),
pilosa.OptServerClusterName(m.Config.Cluster.Name),
@ -691,7 +682,6 @@ func (m *Command) setupServer() error {
OptGRPCServerListener(m.grpcLn),
OptGRPCServerTLSConfig(m.tlsConfig),
OptGRPCServerLogger(m.logger),
OptGRPCServerStats(statsClient),
OptGRPCServerAuth(m.auth),
OptGRPCServerPerm(&p),
OptGRPCServerQueryLogger(m.queryLogger),
@ -828,24 +818,6 @@ func (m *Command) Close() error {
}
}
// newStatsClient creates a stats client from the config
func newStatsClient(name string, host string, namespace string) (stats.StatsClient, error) {
switch name {
case "expvar":
return stats.NewExpvarStatsClient(), nil
case "statsd":
return statsd.NewStatsClient(host, namespace)
case "prometheus":
return prometheus.NewPrometheusClient(
prometheus.OptClientNamespace(namespace),
)
case "nop", "none":
return stats.NopStatsClient, nil
default:
return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, prometheus, none].", name)
}
}
// getListener gets a net.Listener based on the config.
func getListener(uri pnet.URI, tlsconf *tls.Config) (ln net.Listener, err error) {
// If bind URI has the https scheme, enable TLS

6
sql3/interfaces.go
View file

@ -3,6 +3,7 @@ package sql3
import (
"context"
"io"
"github.com/molecula/featurebase/v3/sql3/parser"
"github.com/molecula/featurebase/v3/sql3/planner/types"
@ -10,6 +11,7 @@ import (
type CompilePlanner interface {
CompilePlan(context.Context, parser.Statement) (types.PlanOperator, error)
RehydratePlanOp(context.Context, io.Reader) (types.PlanOperator, error)
}
// Ensure type implements interface.
@ -25,3 +27,7 @@ func NewNopCompilePlanner() *NopCompilePlanner {
func (p *NopCompilePlanner) CompilePlan(ctx context.Context, stmt parser.Statement) (types.PlanOperator, error) {
return nil, nil
}
func (p *NopCompilePlanner) RehydratePlanOp(ctx context.Context, reader io.Reader) (types.PlanOperator, error) {
return nil, nil
}

8
sql3/planner/compilecreatetable.go
View file

@ -196,7 +196,13 @@ func (p *ExecutionPlanner) compileColumn(col *parser.ColumnDefinition) (*createT
case dax.BaseTypeBool:
column.fos = append(column.fos, pilosa.OptFieldTypeBool())
case dax.BaseTypeDecimal:
// Get the scale value.
// if we don't have a scale, it's an error
if col.Type.Scale == nil {
return nil, sql3.NewErrDecimalScaleExpected(col.Type.Name.NamePos.Line, col.Type.Name.NamePos.Column)
}
// get the scale value
scale, err = strconv.ParseInt(col.Type.Scale.Value, 10, 64)
if err != nil {
return nil, err

9
sql3/planner/compileselect.go
View file

@ -302,11 +302,16 @@ func (p *ExecutionPlanner) compileSource(scope *PlanOpQuery, source parser.Sourc
// doing this check here because we don't have a 'system' flag that exists in the FB schema
st, ok := systemTables[strings.ToLower(tableName)]
if ok {
var op types.PlanOperator
op = NewPlanOpSystemTable(p, st)
if st.requiresFanout {
op = NewPlanOpFanout(p, op)
}
if sourceExpr.Alias != nil {
aliasName := parser.IdentName(sourceExpr.Alias)
return NewPlanOpRelAlias(aliasName, NewPlanOpSystemTable(p, st)), nil
return NewPlanOpRelAlias(aliasName, op), nil
}
return NewPlanOpSystemTable(p, st), nil
return op, nil
}
// get all the columns for this table - we will eliminate unused ones

234
sql3/planner/executionplanner.go
View file

@ -3,13 +3,20 @@
package planner
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"strconv"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/sql3"
"github.com/molecula/featurebase/v3/sql3/parser"
"github.com/molecula/featurebase/v3/sql3/planner/types"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
// ExecutionPlanner compiles SQL text into a query plan
@ -79,6 +86,20 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen
return rootOperator, err
}
func (p *ExecutionPlanner) RehydratePlanOp(ctx context.Context, reader io.Reader) (types.PlanOperator, error) {
rdr := newWireProtocolParser(p, reader)
message, err := rdr.nextMessage()
if err != nil {
return nil, err
}
switch m := message.(type) {
case *messagePlanOp:
return m.op, nil
default:
return nil, sql3.NewErrInternalf("unexpected message type '%T'", message)
}
}
func (p *ExecutionPlanner) analyzePlan(stmt parser.Statement) error {
switch stmt := stmt.(type) {
case *parser.SelectStatement:
@ -120,3 +141,216 @@ const (
func (p *ExecutionPlanner) checkAccess(ctx context.Context, objectName string, _ accessType) error {
return nil
}
type reduceFunc func(ctx context.Context, prev, v types.Rows) (types.Rows, error)
type mapResponse struct {
node pilosa.ClusterNode
result types.Rows
err error
}
func (e *ExecutionPlanner) mapReducePlanOp(ctx context.Context, op types.PlanOperator, reduceFn reduceFunc) (result types.Rows, err error) {
ch := make(chan mapResponse)
// Wrap context with a cancel to kill goroutines on exit.
ctx, cancel := context.WithCancel(ctx)
// Create an errgroup so we can wait for all the goroutines to exit
eg, ctx := errgroup.WithContext(ctx)
// After we're done processing, we have to wait for any outstanding
// functions in the ErrGroup to complete. If we didn't have an error
// already at that point, we'll report any errors from the ErrGroup
// instead.
defer func() {
cancel()
errWait := eg.Wait()
if err == nil {
err = errWait
}
}()
nodes := e.systemAPI.ClusterNodes()
// Start mapping across all nodes
if err = e.mapper(ctx, eg, ch, nodes, op, reduceFn); err != nil {
return nil, errors.Wrap(err, "starting mapper")
}
// Iterate over all map responses and reduce.
expected := len(nodes)
done := ctx.Done()
for expected > 0 {
select {
case <-done:
return nil, ctx.Err()
case resp := <-ch:
if resp.err != nil {
return nil, errors.Wrap(resp.err, "query fanout")
}
// if we got a response that we aren't discarding
// because it's an error, subtract it from our count...
expected -= 1
// Reduce value.
result, err = reduceFn(ctx, result, resp.result)
if err != nil {
cancel()
return nil, err
}
}
}
// note the deferred Wait above which might override this nil.
return result, nil
}
func (e *ExecutionPlanner) mapper(ctx context.Context, eg *errgroup.Group, ch chan mapResponse, nodes []pilosa.ClusterNode, op types.PlanOperator, reduceFn reduceFunc) (reterr error) {
done := ctx.Done()
// Execute each node in a separate goroutine.
for _, node := range nodes {
node := node
eg.Go(func() error {
resp := mapResponse{node: node}
// Send local shards to mapper, otherwise remote exec.
if node.ID == e.systemAPI.NodeID() {
iter, err := op.Iterator(ctx, nil)
if err != nil {
resp.result = nil
resp.err = err
}
row, err := iter.Next(ctx)
if err != nil && err != types.ErrNoMoreRows {
resp.result = nil
resp.err = err
}
if err != types.ErrNoMoreRows {
for {
resp.result = append(resp.result, row)
row, err = iter.Next(ctx)
if err != nil && err != types.ErrNoMoreRows {
resp.result = nil
resp.err = err
}
if err == types.ErrNoMoreRows {
break
}
}
}
} else {
results, err := e.remotePlanExec(ctx, node.URI, op)
resp.result = results
resp.err = err
}
// Return response to the channel.
select {
case <-done:
// If someone just canceled the context
// arbitrarily, we could end up here with this
// being the first non-nil error handed to
// the ErrGroup, in which case, it's the best
// explanation we have for why everything's
// stopping.
return ctx.Err()
case ch <- resp:
// If we return a non-nil error from this, the
// entire errGroup gets canceled. So we don't
// want to return a non-nil error if mapReduce
// might try to run another mapper against a
// different set of nodes. Note that this shouldn't
// matter; we just sent the error to mapReduce
// anyway, so it probably cancels the ErrGroup
// too.
if resp.err != nil {
return resp.err
}
}
return nil
})
if reterr != nil {
return reterr // exit early if error occurs when running serially
}
}
return nil
}
func (e *ExecutionPlanner) remotePlanExec(ctx context.Context, addr string, op types.PlanOperator) (types.Rows, error) {
b, err := writeOp(op)
if err != nil {
return nil, err
}
// Create HTTP request.
u := fmt.Sprintf("%s/sql", addr)
req, err := http.NewRequest("POST", u, bytes.NewReader(b))
if err != nil {
return nil, errors.Wrap(err, "creating request")
}
// TODO (pok) internal auth
//AddAuthToken(ctx, &req.Header)
req.Header.Set("Content-Length", strconv.Itoa(len(b)))
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Accept", "application/octet-stream")
req.Header.Set("User-Agent", "pilosa/"+e.systemAPI.Version())
req.Header.Set("X-FeatureBase-Plan-Operator", fmt.Sprintf("%T", op))
// Execute request against the host.
resp, err := http.DefaultClient.Do(req.WithContext(ctx))
if err != nil {
return nil, sql3.NewErrInternalf("error posting internally: %s", err.Error())
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
// we have an error
return nil, sql3.NewErrInternalf("error posting internally: %d", resp.StatusCode)
}
var rows types.Rows
parser := newWireProtocolParser(e, resp.Body)
state := 1
for state <= 2 {
msg, err := parser.nextMessage()
if err != nil {
return nil, err
}
switch state {
case 1:
switch m := msg.(type) {
case *messageSchemaInfo:
parser.schema = m.schema
state = 2
case *messageError:
return nil, m.err
default:
return nil, sql3.NewErrInternalf("unexpected token %d", msg.Token())
}
case 2:
switch m := msg.(type) {
case *messageRow:
rows = append(rows, m.row)
case *messageDone:
// we're done
state = 3
case *messageError:
return nil, m.err
default:
return nil, sql3.NewErrInternalf("unexpected token %d", msg.Token())
}
}
}
return rows, nil
}

13
sql3/planner/opbulkinsert.go
View file

@ -17,6 +17,7 @@ import (
"github.com/PaesslerAG/gval"
"github.com/PaesslerAG/jsonpath"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/sql3"
"github.com/molecula/featurebase/v3/sql3/parser"
@ -367,6 +368,8 @@ func (i *bulkInsertCSVRowIter) Next(ctx context.Context) (types.Row, error) {
return nil, err
}
i.currentBatch = nil
// update the counter for bulk insert batches
pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1)
}
if i.options.rowsLimit > 0 && i.linesRead >= i.options.rowsLimit {
break
@ -378,6 +381,8 @@ func (i *bulkInsertCSVRowIter) Next(ctx context.Context) (types.Row, error) {
return nil, err
}
i.currentBatch = nil
// update the counter for bulk insert batches
pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1)
}
return nil, types.ErrNoMoreRows
}
@ -777,6 +782,8 @@ func (i *bulkInsertNDJsonRowIter) Next(ctx context.Context) (types.Row, error) {
return nil, err
}
i.currentBatch = nil
// update the counter for bulk insert batches
pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1)
}
if i.options.rowsLimit > 0 && i.linesRead >= i.options.rowsLimit {
break
@ -788,6 +795,8 @@ func (i *bulkInsertNDJsonRowIter) Next(ctx context.Context) (types.Row, error) {
return nil, err
}
i.currentBatch = nil
// update the counter for bulk insert batches
pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1)
}
return nil, types.ErrNoMoreRows
}
@ -928,5 +937,9 @@ func processBatch(ctx context.Context, planner *ExecutionPlanner, tableName stri
if err != nil && err != types.ErrNoMoreRows {
return err
}
// update the counter for bulk inserts
pilosa.PerfCounterSQLBulkInsertsSec.Add(int64(len(insertValues)))
return nil
}

114
sql3/planner/opfanout.go Normal file
View file

@ -0,0 +1,114 @@
// Copyright 2022 Molecula Corp. All rights reserved.
package planner
import (
"context"
"fmt"
"github.com/molecula/featurebase/v3/sql3"
"github.com/molecula/featurebase/v3/sql3/planner/types"
)
// PlanOpFanout is a query fanout operator that will execute an operator across all cluster nodes
type PlanOpFanout struct {
planner *ExecutionPlanner
ChildOp types.PlanOperator
warnings []string
}
func NewPlanOpFanout(planner *ExecutionPlanner, child types.PlanOperator) *PlanOpFanout {
return &PlanOpFanout{
planner: planner,
ChildOp: child,
warnings: make([]string, 0),
}
}
func (p *PlanOpFanout) Schema() types.Schema {
return p.ChildOp.Schema()
}
func (p *PlanOpFanout) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) {
return newFanOutIterator(p.planner, p.ChildOp), nil
}
func (p *PlanOpFanout) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) {
if len(children) != 1 {
return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
}
return NewPlanOpFanout(p.planner, children[0]), nil
}
func (p *PlanOpFanout) Children() []types.PlanOperator {
return []types.PlanOperator{
p.ChildOp,
}
}
func (p *PlanOpFanout) Plan() map[string]interface{} {
result := make(map[string]interface{})
result["_op"] = fmt.Sprintf("%T", p)
ps := make([]string, 0)
for _, e := range p.Schema() {
ps = append(ps, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeDescription()))
}
result["_schema"] = ps
result["child"] = p.ChildOp.Plan()
return result
}
func (p *PlanOpFanout) String() string {
return ""
}
func (p *PlanOpFanout) AddWarning(warning string) {
p.warnings = append(p.warnings, warning)
}
func (p *PlanOpFanout) Warnings() []string {
return p.warnings
}
func (p *PlanOpFanout) Expressions() []types.PlanExpression {
return []types.PlanExpression{}
}
func (p *PlanOpFanout) WithUpdatedExpressions(exprs ...types.PlanExpression) (types.PlanOperator, error) {
if len(exprs) != 1 {
return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs))
}
return NewPlanOpFilter(p.planner, exprs[0], p.ChildOp), nil
}
type fanOutIterator struct {
planner *ExecutionPlanner
childOp types.PlanOperator
rows types.Rows
}
func newFanOutIterator(planner *ExecutionPlanner, childOp types.PlanOperator) *fanOutIterator {
return &fanOutIterator{
planner: planner,
childOp: childOp,
}
}
func (i *fanOutIterator) Next(ctx context.Context) (types.Row, error) {
if i.rows == nil {
rows, err := i.planner.mapReducePlanOp(ctx, i.childOp, func(ctx context.Context, prev, v types.Rows) (types.Rows, error) {
return append(prev, v...), nil
})
if err != nil {
return nil, err
}
i.rows = rows
}
if len(i.rows) > 0 {
row := i.rows[0]
// Move to next result element.
i.rows = i.rows[1:]
return row, nil
}
return nil, types.ErrNoMoreRows
}

3
sql3/planner/opinsert.go
View file

@ -368,5 +368,8 @@ func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) {
return nil, errors.Wrap(err, "importing batch")
}
// update the counter for inserts
pilosa.PerfCounterSQLInsertsSec.Add(int64(batch.Len()))
return nil, types.ErrNoMoreRows
}

98
sql3/planner/opsystemtable.go
View file

@ -17,16 +17,18 @@ import (
// exclude this file from SonarCloud dupe eval
const (
fbClusterInfo = "fb_cluster_info"
fbClusterNodes = "fb_cluster_nodes"
fbExecRequests = "fb_exec_requests"
fbClusterInfo = "fb_cluster_info"
fbClusterNodes = "fb_cluster_nodes"
fbExecRequests = "fb_exec_requests"
fbPerformanceCounters = "fb_performance_counters"
fbTableDDL = "fb_table_ddl"
)
type systemTable struct {
name string
schema types.Schema
name string
schema types.Schema
requiresFanout bool
}
var systemTables = map[string]*systemTable{
@ -74,6 +76,7 @@ var systemTables = map[string]*systemTable{
Type: parser.NewDataTypeInt(),
},
},
requiresFanout: false,
},
fbClusterNodes: {
name: fbClusterNodes,
@ -109,11 +112,17 @@ var systemTables = map[string]*systemTable{
Type: parser.NewDataTypeBool(),
},
},
requiresFanout: false,
},
fbExecRequests: {
name: fbExecRequests,
schema: types.Schema{
&types.PlannerColumn{
RelationName: fbPerformanceCounters,
ColumnName: "nodeid",
Type: parser.NewDataTypeString(),
},
&types.PlannerColumn{
RelationName: fbExecRequests,
ColumnName: "request_id",
@ -195,6 +204,7 @@ var systemTables = map[string]*systemTable{
Type: parser.NewDataTypeString(),
},
},
requiresFanout: true,
},
fbTableDDL: {
@ -216,6 +226,44 @@ var systemTables = map[string]*systemTable{
Type: parser.NewDataTypeString(),
},
},
requiresFanout: false,
},
fbPerformanceCounters: {
name: fbPerformanceCounters,
schema: types.Schema{
&types.PlannerColumn{
RelationName: fbPerformanceCounters,
ColumnName: "nodeid",
Type: parser.NewDataTypeString(),
},
&types.PlannerColumn{
RelationName: fbPerformanceCounters,
ColumnName: "namespace",
Type: parser.NewDataTypeString(),
},
&types.PlannerColumn{
RelationName: fbPerformanceCounters,
ColumnName: "subsystem",
Type: parser.NewDataTypeString(),
},
&types.PlannerColumn{
RelationName: fbPerformanceCounters,
ColumnName: "counter_name",
Type: parser.NewDataTypeString(),
},
&types.PlannerColumn{
RelationName: fbPerformanceCounters,
ColumnName: "value",
Type: parser.NewDataTypeInt(),
},
&types.PlannerColumn{
RelationName: fbPerformanceCounters,
ColumnName: "counter_type",
Type: parser.NewDataTypeInt(),
},
},
requiresFanout: true,
},
}
@ -279,6 +327,10 @@ func (p *PlanOpSystemTable) Iterator(ctx context.Context, row types.Row) (types.
return &fbTableDDLRowIter{
planner: p.planner,
}, nil
case fbPerformanceCounters:
return &fbPerformanceCountersRowIter{
planner: p.planner,
}, nil
default:
return nil, sql3.NewErrInternalf("unable to find system table '%s'", p.table.name)
}
@ -367,9 +419,11 @@ func (i *fbExecRequestsRowIter) Next(ctx context.Context) (types.Row, error) {
}
}
nodeId := i.planner.systemAPI.NodeID()
if len(i.result) > 0 {
n := i.result[0]
row := []interface{}{
nodeId,
n.RequestID,
n.UserID,
n.StartTime,
@ -533,3 +587,37 @@ func (i *fbTableDDLRowIter) Next(ctx context.Context) (types.Row, error) {
}
return nil, types.ErrNoMoreRows
}
type fbPerformanceCountersRowIter struct {
planner *ExecutionPlanner
result []pilosa.PerformanceCounter
}
var _ types.RowIterator = (*fbPerformanceCountersRowIter)(nil)
func (i *fbPerformanceCountersRowIter) Next(ctx context.Context) (types.Row, error) {
if i.result == nil {
var err error
i.result, err = pilosa.PerfCounters.ListCounters()
if err != nil {
return nil, err
}
}
nodeId := i.planner.systemAPI.NodeID()
if len(i.result) > 0 {
n := i.result[0]
row := []interface{}{
nodeId,
n.NameSpace,
n.SubSystem,
n.CounterName,
n.Value,
n.CounterType,
}
// Move to next result element.
i.result = i.result[1:]
return row, nil
}
return nil, types.ErrNoMoreRows
}

5
sql3/planner/types/operator.go
View file

@ -80,9 +80,12 @@ func (r Schema) Plan() []map[string]interface{} {
return result
}
// Row is a tuple of values
// Row is a tuple (of values)
type Row []interface{}
// Rows is a table of rows
type Rows []Row
// Append appends all the values in r2 to this row and returns the result
func (r Row) Append(r2 Row) Row {
row := make(Row, len(r)+len(r2))

211
sql3/planner/wireprotocol.go Normal file
View file

@ -0,0 +1,211 @@
package planner
import (
"bufio"
"bytes"
"encoding/binary"
"errors"
"io"
"github.com/molecula/featurebase/v3/sql3"
"github.com/molecula/featurebase/v3/sql3/planner/types"
"github.com/molecula/featurebase/v3/wireprotocol"
)
type wireProtocolMessage interface {
Token() int16
}
func writeOp(op types.PlanOperator) ([]byte, error) {
buf := new(bytes.Buffer)
writer := bufio.NewWriter(buf)
// serialize a plan op - for now we are just supporting
// system tables, and we'll send the name of the system table
switch op := op.(type) {
case *PlanOpSystemTable:
// write token
b := make([]byte, 2)
binary.BigEndian.PutUint16(b, uint16(wireprotocol.TOKEN_PLAN_OP))
writer.Write(b)
// write table name len
t := op.table.name
b = make([]byte, 4)
binary.BigEndian.PutUint32(b, uint32(len(t)))
writer.Write(b)
// write table name
writer.WriteString(op.table.name)
writer.Flush()
default:
return []byte{}, sql3.NewErrInternalf("unexpected plan operator type '%T'", op)
}
return buf.Bytes(), nil
}
type wireProtocolParser struct {
planner *ExecutionPlanner
reader io.Reader
schema types.Schema
}
func newWireProtocolParser(p *ExecutionPlanner, reader io.Reader) *wireProtocolParser {
return &wireProtocolParser{
planner: p,
reader: reader,
}
}
func (f *wireProtocolParser) nextMessage() (wireProtocolMessage, error) {
var t int16
err := binary.Read(f.reader, binary.BigEndian, &t)
if err != nil {
return nil, err
}
switch t {
case wireprotocol.TOKEN_SCHEMA_INFO:
return newMessageSchemaInfo(f.planner, f.reader)
case wireprotocol.TOKEN_ROW:
if f.schema == nil {
return nil, sql3.NewErrInternalf("schema uninitialized")
}
return newMessageRow(f.planner, f.reader, f.schema)
case wireprotocol.TOKEN_ERROR_MESSAGE:
return newMessageError(f.planner, f.reader)
case wireprotocol.TOKEN_DONE:
return newMessageDone(f.planner, f.reader)
case wireprotocol.TOKEN_PLAN_OP:
return newMessagePlanOp(f.planner, f.reader)
default:
return nil, sql3.NewErrInternalf("unexpected token %d", t)
}
}
type messagePlanOp struct {
token int16
op types.PlanOperator
}
var _ wireProtocolMessage = (*messagePlanOp)(nil)
func newMessagePlanOp(p *ExecutionPlanner, reader io.Reader) (*messagePlanOp, error) {
var len int32
err := binary.Read(reader, binary.BigEndian, &len)
if err != nil {
return nil, err
}
bname := make([]byte, len)
err = binary.Read(reader, binary.BigEndian, &bname)
if err != nil {
return nil, err
}
name := string(bname)
st, ok := systemTables[name]
if !ok {
return nil, sql3.NewErrInternalf("unexpected system table name %s", name)
}
return &messagePlanOp{
token: wireprotocol.TOKEN_PLAN_OP,
op: NewPlanOpSystemTable(p, st),
}, nil
}
func (m *messagePlanOp) Token() int16 {
return m.token
}
type messageError struct {
token int16
err error
}
var _ wireProtocolMessage = (*messageError)(nil)
func newMessageError(p *ExecutionPlanner, reader io.Reader) (*messageError, error) {
var len int32
err := binary.Read(reader, binary.BigEndian, &len)
if err != nil {
return nil, err
}
bname := make([]byte, len)
err = binary.Read(reader, binary.BigEndian, &bname)
if err != nil {
return nil, err
}
errMsg := string(bname)
return &messageError{
token: wireprotocol.TOKEN_ERROR_MESSAGE,
err: errors.New(errMsg),
}, nil
}
func (m *messageError) Token() int16 {
return m.token
}
type messageSchemaInfo struct {
token int16
schema types.Schema
}
var _ wireProtocolMessage = (*messageSchemaInfo)(nil)
func newMessageSchemaInfo(p *ExecutionPlanner, reader io.Reader) (*messageSchemaInfo, error) {
schema, err := wireprotocol.ReadSchema(reader)
if err != nil {
return nil, err
}
return &messageSchemaInfo{
token: wireprotocol.TOKEN_SCHEMA_INFO,
schema: schema,
}, nil
}
func (m *messageSchemaInfo) Token() int16 {
return m.token
}
type messageRow struct {
token int16
row types.Row
}
var _ wireProtocolMessage = (*messageRow)(nil)
func newMessageRow(p *ExecutionPlanner, reader io.Reader, schema types.Schema) (*messageRow, error) {
row, err := wireprotocol.ReadRow(reader, schema)
if err != nil {
return nil, err
}
return &messageRow{
token: wireprotocol.TOKEN_ROW,
row: row,
}, nil
}
func (m *messageRow) Token() int16 {
return m.token
}
type messageDone struct {
token int16
}
var _ wireProtocolMessage = (*messageDone)(nil)
func newMessageDone(p *ExecutionPlanner, reader io.Reader) (*messageDone, error) {
return &messageDone{
token: wireprotocol.TOKEN_DONE,
}, nil
}
func (m *messageDone) Token() int16 {
return m.token
}

140
sql3/sql_complex_test.go
View file

@ -36,6 +36,92 @@ func TestPlanner_Misc(t *testing.T) {
assert.True(t, d.EqualTo(pql.NewDecimal(12345678, 6)))
}
func TestPlanner_SystemTableFanout(t *testing.T) {
c := test.MustRunCluster(t, 3)
defer c.Close()
server := c.GetNode(0).Server
t.Run("PerfCounters", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, server, `select * from fb_performance_counters`)
if err != nil {
t.Fatal(err)
}
if len(results) != 15 {
t.Fatal(fmt.Errorf("unexpected result set length"))
}
if diff := cmp.Diff([]*pilosa.WireQueryField{
wireQueryFieldString("nodeid"),
wireQueryFieldString("namespace"),
wireQueryFieldString("subsystem"),
wireQueryFieldString("counter_name"),
wireQueryFieldInt("value"),
wireQueryFieldInt("counter_type"),
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("SystemTablesExecRequests", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select * from fb_exec_requests`)
if err != nil {
t.Fatal(err)
}
if len(results) != 2 {
t.Fatal(fmt.Errorf("unexpected result set length"))
}
if diff := cmp.Diff([]*pilosa.WireQueryField{
wireQueryFieldString("nodeid"),
wireQueryFieldString("request_id"),
wireQueryFieldString("user"),
wireQueryFieldTimestamp("start_time"),
wireQueryFieldTimestamp("end_time"),
wireQueryFieldString("status"),
wireQueryFieldString("wait_type"),
wireQueryFieldInt("wait_time"),
wireQueryFieldString("wait_resource"),
wireQueryFieldInt("cpu_time"),
wireQueryFieldInt("elapsed_time"),
wireQueryFieldInt("reads"),
wireQueryFieldInt("writes"),
wireQueryFieldInt("logical_reads"),
wireQueryFieldInt("row_count"),
wireQueryFieldString("sql"),
wireQueryFieldString("plan"),
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("SystemTablesExecRequestsAgg", func(t *testing.T) {
_, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select
count(request_id) as request_count,
min(elapsed_time) as min_duration,
max(elapsed_time) as max_duration,
avg(elapsed_time) as avg_duration
from
fb_exec_requests
where
status = 'complete';`)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff([]*pilosa.WireQueryField{
wireQueryFieldInt("request_count"),
wireQueryFieldInt("min_duration"),
wireQueryFieldInt("max_duration"),
wireQueryFieldDecimal("avg_duration", 4),
}, columns); diff != "" {
t.Fatal(diff)
}
})
}
func TestPlanner_Show(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
@ -102,64 +188,12 @@ func TestPlanner_Show(t *testing.T) {
}
})
t.Run("SystemTablesExecRequests", func(t *testing.T) {
_, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select * from fb_exec_requests`)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff([]*pilosa.WireQueryField{
wireQueryFieldString("request_id"),
wireQueryFieldString("user"),
wireQueryFieldTimestamp("start_time"),
wireQueryFieldTimestamp("end_time"),
wireQueryFieldString("status"),
wireQueryFieldString("wait_type"),
wireQueryFieldInt("wait_time"),
wireQueryFieldString("wait_resource"),
wireQueryFieldInt("cpu_time"),
wireQueryFieldInt("elapsed_time"),
wireQueryFieldInt("reads"),
wireQueryFieldInt("writes"),
wireQueryFieldInt("logical_reads"),
wireQueryFieldInt("row_count"),
wireQueryFieldString("sql"),
wireQueryFieldString("plan"),
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("SystemTablesExecRequestsAgg", func(t *testing.T) {
_, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select
count(request_id) as request_count,
min(elapsed_time) as min_duration,
max(elapsed_time) as max_duration,
avg(elapsed_time) as avg_duration
from
fb_exec_requests
where
status = 'complete';`)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff([]*pilosa.WireQueryField{
wireQueryFieldInt("request_count"),
wireQueryFieldInt("min_duration"),
wireQueryFieldInt("max_duration"),
wireQueryFieldDecimal("avg_duration", 4),
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("ShowTables", func(t *testing.T) {
results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SHOW TABLES`)
if err != nil {
t.Fatal(err)
}
if len(results) != 6 {
if len(results) != 7 {
t.Fatal(fmt.Errorf("unexpected result set length"))
}

273
stats/stats.go
View file

@ -1,273 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package stats
import (
"expvar"
"sort"
"strings"
"sync"
"time"
"github.com/molecula/featurebase/v3/logger"
)
// Expvar global expvar map.
var Expvar *expvar.Map
// StatsClient represents a client to a stats server.
type StatsClient interface {
// Returns a sorted list of tags on the client.
Tags() []string
// Returns a new client with additional tags appended.
WithTags(tags ...string) StatsClient
// Tracks the number of times something occurs per second.
Count(name string, value int64, rate float64)
// Tracks the number of times something occurs per second with custom tags
CountWithCustomTags(name string, value int64, rate float64, tags []string)
// Sets the value of a metric.
Gauge(name string, value float64, rate float64)
// Tracks statistical distribution of a metric.
Histogram(name string, value float64, rate float64)
// Tracks number of unique elements.
Set(name string, value string, rate float64)
// Tracks timing information for a metric.
Timing(name string, value time.Duration, rate float64)
// SetLogger Set the logger output type
SetLogger(logger logger.Logger)
// Starts the service
Open()
// Closes the client
Close() error
}
// NopStatsClient represents a client that doesn't do anything.
var NopStatsClient StatsClient = &nopStatsClient{}
type nopStatsClient struct{}
func (c *nopStatsClient) Tags() []string { return nil }
func (c *nopStatsClient) WithTags(tags ...string) StatsClient { return c }
func (c *nopStatsClient) Count(name string, value int64, rate float64) {}
func (c *nopStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) {}
func (c *nopStatsClient) Gauge(name string, value float64, rate float64) {}
func (c *nopStatsClient) Histogram(name string, value float64, rate float64) {}
func (c *nopStatsClient) Set(name string, value string, rate float64) {}
func (c *nopStatsClient) Timing(name string, value time.Duration, rate float64) {}
func (c *nopStatsClient) SetLogger(logger logger.Logger) {}
func (c *nopStatsClient) Open() {}
func (c *nopStatsClient) Close() error { return nil }
// expvarStatsClient writes stats out to expvars.
type expvarStatsClient struct {
mu sync.Mutex
m *expvar.Map
tags []string
}
// NewExpvarStatsClient returns a new instance of ExpvarStatsClient.
// This client points at the root of the expvar index map.
func NewExpvarStatsClient() *expvarStatsClient {
if Expvar == nil {
Expvar = expvar.NewMap("index")
}
return &expvarStatsClient{
m: Expvar,
}
}
// Tags returns a sorted list of tags on the client.
func (c *expvarStatsClient) Tags() []string {
return nil
}
// WithTags returns a new client with additional tags appended.
func (c *expvarStatsClient) WithTags(tags ...string) StatsClient {
m := &expvar.Map{}
m.Init()
c.m.Set(strings.Join(tags, ","), m)
return &expvarStatsClient{
m: m,
tags: unionStringSlice(c.tags, tags),
}
}
// Count tracks the number of times something occurs.
func (c *expvarStatsClient) Count(name string, value int64, rate float64) {
c.m.Add(name, value)
}
// CountWithCustomTags Tracks the number of times something occurs per second with custom tags
func (c *expvarStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) {
c.m.Add(name, value)
}
// Gauge sets the value of a metric.
func (c *expvarStatsClient) Gauge(name string, value float64, rate float64) {
var f expvar.Float
f.Set(value)
c.m.Set(name, &f)
}
// Histogram tracks statistical distribution of a metric.
// This works the same as gauge for this client.
func (c *expvarStatsClient) Histogram(name string, value float64, rate float64) {
c.Gauge(name, value, rate)
}
// Set tracks number of unique elements.
func (c *expvarStatsClient) Set(name string, value string, rate float64) {
var s expvar.String
s.Set(value)
c.m.Set(name, &s)
}
// Timing tracks timing information for a metric.
func (c *expvarStatsClient) Timing(name string, value time.Duration, rate float64) {
c.mu.Lock()
d, _ := c.m.Get(name).(time.Duration)
c.m.Set(name, d+value)
c.mu.Unlock()
}
// SetLogger has no logger.
func (c *expvarStatsClient) SetLogger(logger logger.Logger) {
}
// Open no-op.
func (c *expvarStatsClient) Open() {}
// Close no-op.
func (c *expvarStatsClient) Close() error { return nil }
// MultiStatsClient joins multiple stats clients together.
type MultiStatsClient []StatsClient
// Tags returns tags from the first client.
func (a MultiStatsClient) Tags() []string {
if len(a) > 0 {
return a[0].Tags()
}
return nil
}
// WithTags returns a new set of clients with the additional tags.
func (a MultiStatsClient) WithTags(tags ...string) StatsClient {
other := make(MultiStatsClient, len(a))
for i := range a {
other[i] = a[i].WithTags(tags...)
}
return other
}
// Count tracks the number of times something occurs per second on all clients.
func (a MultiStatsClient) Count(name string, value int64, rate float64) {
for _, c := range a {
c.Count(name, value, rate)
}
}
// CountWithCustomTags Tracks the number of times something occurs per second with custom tags
func (a MultiStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) {
for _, c := range a {
c.CountWithCustomTags(name, value, rate, tags)
}
}
// Gauge sets the value of a metric on all clients.
func (a MultiStatsClient) Gauge(name string, value float64, rate float64) {
for _, c := range a {
c.Gauge(name, value, rate)
}
}
// Histogram tracks statistical distribution of a metric on all clients.
func (a MultiStatsClient) Histogram(name string, value float64, rate float64) {
for _, c := range a {
c.Histogram(name, value, rate)
}
}
// Set tracks number of unique elements on all clients.
func (a MultiStatsClient) Set(name string, value string, rate float64) {
for _, c := range a {
c.Set(name, value, rate)
}
}
// Timing tracks timing information for a metric on all clients.
func (a MultiStatsClient) Timing(name string, value time.Duration, rate float64) {
for _, c := range a {
c.Timing(name, value, rate)
}
}
// SetLogger Sets the StatsD logger output type.
func (a MultiStatsClient) SetLogger(logger logger.Logger) {
for _, c := range a {
c.SetLogger(logger)
}
}
// Open starts the stat service.
func (a MultiStatsClient) Open() {
for _, c := range a {
c.Open()
}
}
// Close shuts down the stats clients.
func (a MultiStatsClient) Close() error {
for _, c := range a {
err := c.Close()
if err != nil {
return err
}
}
return nil
}
// unionStringSlice returns a sorted set of tags which combine a & b.
func unionStringSlice(a, b []string) []string {
// Sort both sets first.
sort.Strings(a)
sort.Strings(b)
// Find size of largest slice.
n := len(a)
if len(b) > n {
n = len(b)
}
// Exit if both sets are empty.
if n == 0 {
return nil
}
// Iterate over both in order and merge.
other := make([]string, 0, n)
for len(a) > 0 || len(b) > 0 {
if len(a) == 0 {
other, b = append(other, b[0]), b[1:]
} else if len(b) == 0 {
other, a = append(other, a[0]), a[1:]
} else if a[0] < b[0] {
other, a = append(other, a[0]), a[1:]
} else if b[0] < a[0] {
other, b = append(other, b[0]), b[1:]
} else {
other, a, b = append(other, a[0]), a[1:], b[1:]
}
}
return other
}

257
stats/stats_test.go
View file

@ -1,257 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package stats_test
import (
"context"
"fmt"
"net/http/httptest"
"strings"
"testing"
"time"
pilosa "github.com/molecula/featurebase/v3"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/test"
)
// TestMultiStatClient_Expvar run the multistat client with exp var
// since the EXPVAR data is stored in a global we should run these in one test function
func TestMultiStatClient_Expvar(t *testing.T) {
hldr := test.MustOpenHolder(t)
c := stats.NewExpvarStatsClient()
ms := make(stats.MultiStatsClient, 1)
ms[0] = c
hldr.Stats = ms
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
hldr.SetBit("d", "f", 0, pilosa.ShardWidth)
hldr.SetBit("d", "f", 0, pilosa.ShardWidth+2)
hldr.ClearBit("d", "f", 0, 1)
indexStats := fmt.Sprintf(`{"%s": %d, "%s": %d}`, pilosa.MetricClearBit, 1, pilosa.MetricSetBit, 4)
if stats.Expvar.String() != `{"index:d": `+indexStats+`}` {
t.Fatalf("unexpected expvar : %s", stats.Expvar.String())
}
hldr.Stats.CountWithCustomTags("cc", 1, 1.0, []string{"foo:bar"})
if stats.Expvar.String() != `{"cc": 1, "index:d": `+indexStats+`}` {
t.Fatalf("unexpected expvar : %s", stats.Expvar.String())
}
// Gauge creates a unique key, subsequent Gauge calls will overwrite
hldr.Stats.Gauge("g", 5, 1.0)
hldr.Stats.Gauge("g", 8, 1.0)
if stats.Expvar.String() != `{"cc": 1, "g": 8, "index:d": `+indexStats+`}` {
t.Fatalf("unexpected expvar : %s", stats.Expvar.String())
}
// Set creates a unique key, subsequent sets will overwrite
hldr.Stats.Set("s", "4", 1.0)
hldr.Stats.Set("s", "7", 1.0)
if stats.Expvar.String() != `{"cc": 1, "g": 8, "index:d": `+indexStats+`, "s": "7"}` {
t.Fatalf("unexpected expvar : %s", stats.Expvar.String())
}
// Record timing duration and a uniquely Set key/value
dur, _ := time.ParseDuration("123us")
hldr.Stats.Timing("tt", dur, 1.0)
if stats.Expvar.String() != `{"cc": 1, "g": 8, "index:d": `+indexStats+`, "s": "7", "tt": 123µs}` {
t.Fatalf("unexpected expvar : %s", stats.Expvar.String())
}
// Expvar histogram is implemented as a gauge
hldr.Stats.Histogram("hh", 3, 1.0)
if stats.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": `+indexStats+`, "s": "7", "tt": 123µs}` {
t.Fatalf("unexpected expvar : %s", stats.Expvar.String())
}
// Expvar should ignore earlier set tags from setbit
if hldr.Stats.Tags() != nil {
t.Fatalf("unexpected tag")
}
}
func TestStatsCount_TopN(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
// Execute query.
called := false
hldr.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != "query_topn_total" {
t.Errorf("Expected query_topn_total, Results %s", name)
}
if tags[0] != "index:d" {
t.Errorf("Expected index, Results %s", tags[0])
}
called = true
},
}
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
hldr.SetBit("d", "f", 0, pilosa.ShardWidth)
hldr.SetBit("d", "f", 0, pilosa.ShardWidth+2)
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `TopN(field=f, n=2)`}); err != nil {
t.Fatal(err)
}
if !called {
t.Error("CountWithCustomTags name isn't called")
}
}
func TestStatsCount_Bitmap(t *testing.T) {
// Cluster has to be unhsared because we're mocking the stats which writes
// to a holder in use by other tests.
c := test.MustRunUnsharedCluster(t, 1)
defer c.Close()
hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
called := false
hldr.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != pilosa.MetricRow {
t.Errorf("Expected %s, Results %s", pilosa.MetricRow, name)
}
if tags[0] != "index:d" {
t.Errorf("Expected index, Results %s", tags[0])
}
called = true
},
}
hldr.SetBit("d", "f", 0, 0)
hldr.SetBit("d", "f", 0, 1)
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `Row(f=0)`}); err != nil {
t.Fatal(err)
}
if !called {
t.Error("CountWithCustomTags name isn't called")
}
}
func TestStatsCount_APICalls(t *testing.T) {
// We can't share a cluster when we're modifying its stats counter.
cluster := test.MustRunUnsharedCluster(t, 1)
defer cluster.Close()
cmd := cluster.GetNode(0)
h := cmd.Handler.(*pilosa.Handler).Handler
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
t.Run("create index", func(t *testing.T) {
called := false
hldr.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != pilosa.MetricCreateIndex {
t.Errorf("Expected %v, Results %s", pilosa.MetricCreateIndex, name)
}
called = true
},
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
})
t.Run("create field", func(t *testing.T) {
called := false
hldr.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != pilosa.MetricCreateField {
t.Errorf("Expected %v, Results %s", pilosa.MetricCreateField, name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
}
called = true
},
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/field/f", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
})
t.Run("delete field", func(t *testing.T) {
called := false
hldr.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != pilosa.MetricDeleteField {
t.Errorf("Expected %v, Results %s", pilosa.MetricDeleteField, name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
}
called = true
},
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
})
t.Run("delete index", func(t *testing.T) {
called := false
hldr.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != pilosa.MetricDeleteIndex {
t.Errorf("Expected %v, Results %s", pilosa.MetricDeleteIndex, name)
}
called = true
},
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
})
}
type MockStats struct {
mockCount func(name string, value int64, rate float64)
mockCountWithTags func(name string, value int64, rate float64, tags []string)
}
func (s *MockStats) Count(name string, value int64, rate float64) {
if s.mockCount != nil {
s.mockCount(name, value, rate)
}
}
func (s *MockStats) CountWithCustomTags(name string, value int64, rate float64, tags []string) {
if s.mockCountWithTags != nil {
s.mockCountWithTags(name, value, rate, tags)
}
}
func (c *MockStats) Tags() []string { return nil }
func (c *MockStats) WithTags(tags ...string) stats.StatsClient { return c }
func (c *MockStats) Gauge(name string, value float64, rate float64) {}
func (c *MockStats) Histogram(name string, value float64, rate float64) {}
func (c *MockStats) Set(name string, value string, rate float64) {}
func (c *MockStats) Timing(name string, value time.Duration, rate float64) {}
func (c *MockStats) SetLogger(logger logger.Logger) {}
func (c *MockStats) Open() {}
func (c *MockStats) Close() error { return nil }

151
statsd/statsd.go
View file

@ -1,151 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package statsd
import (
"sort"
"time"
"github.com/DataDog/datadog-go/statsd"
"github.com/molecula/featurebase/v3/logger"
"github.com/molecula/featurebase/v3/stats"
)
// StatsD protocol wrapper using the DataDog library that added Tags to the StatsD protocol
// statsD defailt host is "127.0.0.1:8125"
const (
// bufferLen Stats lient buffer size.
bufferLen = 1024
)
// Ensure client implements interface.
var _ stats.StatsClient = &statsClient{}
// statsClient represents a StatsD implementation of pilosa.statsClient.
type statsClient struct {
client *statsd.Client
tags []string
logger logger.Logger
// prefix is appended to each metric event name
prefix string
}
// NewStatsClient returns a new instance of StatsClient.
func NewStatsClient(host string, namespace string) (*statsClient, error) {
c, err := statsd.NewBuffered(host, bufferLen)
if err != nil {
return nil, err
}
return &statsClient{
client: c,
logger: logger.NopLogger,
prefix: namespace + ".",
}, nil
}
// Open no-op
func (c *statsClient) Open() {}
// Close closes the connection to the agent.
func (c *statsClient) Close() error {
return c.client.Close()
}
// Tags returns a sorted list of tags on the client.
func (c *statsClient) Tags() []string {
return c.tags
}
// WithTags returns a new client with additional tags appended.
func (c *statsClient) WithTags(tags ...string) stats.StatsClient {
return &statsClient{
client: c.client,
tags: unionStringSlice(c.tags, tags),
logger: c.logger,
}
}
// Count tracks the number of times something occurs per second.
func (c *statsClient) Count(name string, value int64, rate float64) {
if err := c.client.Count(c.prefix+name, value, c.tags, rate); err != nil {
c.logger.Errorf("statsd.StatsClient.Count error: %s", err)
}
}
// CountWithCustomTags tracks the number of times something occurs per second with custom tags.
func (c *statsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) {
tags := append(c.tags, t...)
if err := c.client.Count(c.prefix+name, value, tags, rate); err != nil {
c.logger.Errorf("statsd.StatsClient.Count error: %s", err)
}
}
// Gauge sets the value of a metric.
func (c *statsClient) Gauge(name string, value float64, rate float64) {
if err := c.client.Gauge(c.prefix+name, value, c.tags, rate); err != nil {
c.logger.Errorf("statsd.StatsClient.Gauge error: %s", err)
}
}
// Histogram tracks statistical distribution of a metric.
func (c *statsClient) Histogram(name string, value float64, rate float64) {
if err := c.client.Histogram(c.prefix+name, value, c.tags, rate); err != nil {
c.logger.Errorf("statsd.StatsClient.Histogram error: %s", err)
}
}
// Set tracks number of unique elements.
func (c *statsClient) Set(name string, value string, rate float64) {
if err := c.client.Set(c.prefix+name, value, c.tags, rate); err != nil {
c.logger.Errorf("statsd.StatsClient.Set error: %s", err)
}
}
// Timing tracks timing information for a metric.
func (c *statsClient) Timing(name string, value time.Duration, rate float64) {
if err := c.client.Timing(c.prefix+name, value, c.tags, rate); err != nil {
c.logger.Errorf("statsd.StatsClient.Timing error: %s", err)
}
}
// SetLogger sets the logger for client.
func (c *statsClient) SetLogger(logger logger.Logger) {
c.logger = logger
}
// unionStringSlice returns a sorted set of tags which combine a & b.
func unionStringSlice(a, b []string) []string {
// Sort both sets first.
sort.Strings(a)
sort.Strings(b)
// Find size of largest slice.
n := len(a)
if len(b) > n {
n = len(b)
}
// Exit if both sets are empty.
if n == 0 {
return nil
}
// Iterate over both in order and merge.
other := make([]string, 0, n)
for len(a) > 0 || len(b) > 0 {
if len(a) == 0 {
other, b = append(other, b[0]), b[1:]
} else if len(b) == 0 {
other, a = append(other, a[0]), a[1:]
} else if a[0] < b[0] {
other, a = append(other, a[0]), a[1:]
} else if b[0] < a[0] {
other, b = append(other, b[0]), b[1:]
} else {
other, a, b = append(other, a[0]), a[1:], b[1:]
}
}
return other
}

49
statsd/statsd_test.go
View file

@ -1,49 +0,0 @@
// Copyright 2021 Molecula Corp. All rights reserved.
package statsd_test
import (
"reflect"
"testing"
"time"
"github.com/molecula/featurebase/v3/statsd"
_ "github.com/molecula/featurebase/v3/test"
)
func TestStatsClient_WithTags(t *testing.T) {
// Create a new client.
c, err := statsd.NewStatsClient("localhost:19444", "testnamespace")
if err != nil {
t.Fatal(err)
}
defer c.Close()
// Create a new client with additional tags.
c1 := c.WithTags("foo", "bar")
if tags := c1.Tags(); !reflect.DeepEqual(tags, []string{"bar", "foo"}) {
t.Fatalf("unexpected tags: %+v", tags)
}
// Create a new client from the clone with more tags.
c2 := c1.WithTags("bar", "baz")
if tags := c2.Tags(); !reflect.DeepEqual(tags, []string{"bar", "baz", "foo"}) {
t.Fatalf("unexpected tags: %+v", tags)
}
}
func TestStatsClient_Methods(t *testing.T) {
// Create a new client.
c, err := statsd.NewStatsClient("localhost:19444", "testnamespace")
if err != nil {
t.Fatal(err)
}
defer c.Close()
dur, _ := time.ParseDuration("123us")
c.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"})
c.Count("cc", 1, 1.0)
c.Gauge("gg", 10, 1.0)
c.Histogram("hh", 1, 1.0)
c.Timing("tt", dur, 1.0)
c.Set("ss", "ss", 1.0)
}

4
view.go
View file

@ -15,7 +15,6 @@ import (
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/roaring"
"github.com/molecula/featurebase/v3/stats"
"github.com/molecula/featurebase/v3/testhook"
"github.com/molecula/featurebase/v3/vprint"
"github.com/pkg/errors"
@ -50,7 +49,6 @@ type view struct {
fragments map[uint64]*fragment
broadcaster broadcaster
stats stats.StatsClient
knownShards *roaring.Bitmap
knownShardsCopied uint32
@ -78,7 +76,6 @@ func newView(holder *Holder, path, index, field, name string, fieldOptions Field
fragments: make(map[uint64]*fragment),
broadcaster: NopBroadcaster,
stats: stats.NopStatsClient,
knownShards: roaring.NewSliceBitmap(),
closing: make(chan struct{}),
@ -392,7 +389,6 @@ func (v *view) newFragment(shard uint64) *fragment {
frag := newFragment(v.holder, v.idx, v.fld, v, shard)
frag.CacheType = v.cacheType
frag.CacheSize = v.cacheSize
frag.stats = v.stats
if v.fieldType == FieldTypeMutex {
frag.mutexVector = newRowsVector(frag)
} else if v.fieldType == FieldTypeBool {

563
wireprotocol/wireprimitives.go Normal file
View file

@ -0,0 +1,563 @@
package wireprotocol
import (
"bufio"
"bytes"
"encoding/binary"
"time"
"io"
"github.com/molecula/featurebase/v3/errors"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/sql3/parser"
"github.com/molecula/featurebase/v3/sql3/planner/types"
)
const (
// server --> client
TOKEN_SCHEMA_INFO int16 = 0xA1
TOKEN_ROW int16 = 0xA2
TOKEN_DONE int16 = 0xFD
TOKEN_INFO_MESSAGE int16 = 0xFE
TOKEN_ERROR_MESSAGE int16 = 0xFF
// client --> server
TOKEN_SQL int16 = 0x01
TOKEN_PLAN_OP int16 = 0x02
)
const (
TYPE_VOID int8 = 0x00
TYPE_ID int8 = 0x01
TYPE_BOOL int8 = 0x02
TYPE_INT int8 = 0x03
TYPE_DECIMAL int8 = 0x04
TYPE_TIMESTAMP int8 = 0x05
TYPE_IDSET int8 = 0x06
TYPE_STRING int8 = 0x07
TYPE_STRINGSET int8 = 0x08
)
func ExpectToken(reader io.Reader, token int16) (int16, error) {
var tk int16
err := binary.Read(reader, binary.BigEndian, &tk)
if err != nil {
return 0, err
}
if tk != token {
return 0, errors.Errorf("expected token found %d", token)
}
return tk, nil
}
// TOKEN_COLUMN_INFO message
// length (bytes)
// token 2
// column count 2
//
// (n) columns
//
// name length 1
// name (from prev)
// data type 1
//
// (optional)
// if type decimal
// scale 1
// note RelationName and AliasName members from
// PlannerColumn are not sent over the wire
func WriteSchema(schema types.Schema) ([]byte, error) {
buf := new(bytes.Buffer)
writer := bufio.NewWriter(buf)
// write token
writeToken(writer, TOKEN_SCHEMA_INFO)
// column count
writeInt16(writer, int16(len(schema)))
// for each column
for _, s := range schema {
// name len byte
writeInt8(writer, int8(len(s.ColumnName)))
// name
writer.WriteString(s.ColumnName)
// type byte
switch ty := s.Type.(type) {
case *parser.DataTypeID:
writeInt8(writer, TYPE_ID)
case *parser.DataTypeBool:
writeInt8(writer, TYPE_BOOL)
case *parser.DataTypeInt:
writeInt8(writer, TYPE_INT)
case *parser.DataTypeDecimal:
writeInt8(writer, TYPE_DECIMAL)
writeInt8(writer, int8(ty.Scale))
case *parser.DataTypeTimestamp:
writeInt8(writer, TYPE_TIMESTAMP)
case *parser.DataTypeIDSet:
writeInt8(writer, TYPE_IDSET)
case *parser.DataTypeString:
writeInt8(writer, TYPE_STRING)
case *parser.DataTypeStringSet:
writeInt8(writer, TYPE_STRINGSET)
default:
return []byte{}, errors.Errorf("unexpected type '%T'", s.Type)
}
}
writer.Flush()
return buf.Bytes(), nil
}
func ReadSchema(reader io.Reader) (types.Schema, error) {
var columnCount int16
err := binary.Read(reader, binary.BigEndian, &columnCount)
if err != nil {
return nil, err
}
var schema types.Schema
for i := 0; i < int(columnCount); i++ {
var nameLen int8
err = binary.Read(reader, binary.BigEndian, &nameLen)
if err != nil {
return nil, err
}
bname := make([]byte, nameLen)
err = binary.Read(reader, binary.BigEndian, &bname)
if err != nil {
return nil, err
}
colName := string(bname)
var typ int8
err = binary.Read(reader, binary.BigEndian, &typ)
if err != nil {
return nil, err
}
var dataType parser.ExprDataType
switch typ {
case TYPE_ID:
dataType = parser.NewDataTypeID()
case TYPE_BOOL:
dataType = parser.NewDataTypeBool()
case TYPE_INT:
dataType = parser.NewDataTypeInt()
case TYPE_DECIMAL:
var scale int8
err = binary.Read(reader, binary.BigEndian, &scale)
if err != nil {
return nil, err
}
dataType = parser.NewDataTypeDecimal(int64(scale))
case TYPE_TIMESTAMP:
dataType = parser.NewDataTypeTimestamp()
case TYPE_IDSET:
dataType = parser.NewDataTypeIDSet()
case TYPE_STRING:
dataType = parser.NewDataTypeString()
case TYPE_STRINGSET:
dataType = parser.NewDataTypeStringSet()
}
schema = append(schema, &types.PlannerColumn{
ColumnName: colName,
Type: dataType,
})
}
return schema, nil
}
// TOKEN_ROW message
// length (bytes)
// token 2
//
// (n) columns
// if column length is 0 --> null
//
// - for ID, INT
//
// column length 1
// value 8
//
// - for DECIMAL
//
// column length 1
// value 8
//
// - for BOOL
//
// column length 1
// value 1
//
// - for TIMESTAMP
//
// column length 1
// value 8
//
// - for IDSET
//
// set length 2
// (n) values
// value 8
//
// - for STRING
//
// column length 2
// value (from prev)
//
// - for STRINGSET
//
// set length 2
// (n) values
// value len 2
// value (from prev)
//
func WriteRow(row types.Row, schema types.Schema) ([]byte, error) {
buf := new(bytes.Buffer)
writer := bufio.NewWriter(buf)
// write token
writeToken(writer, TOKEN_ROW)
// for each column
for i, s := range schema {
val := row[i]
switch s.Type.(type) {
case *parser.DataTypeID, *parser.DataTypeInt:
if val == nil {
writeInt8(writer, 0)
} else {
writeInt8(writer, 8)
v, ok := row[i].(int64)
if !ok {
return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
}
writeInt64(writer, v)
}
case *parser.DataTypeDecimal:
if val == nil {
writeInt8(writer, 0)
} else {
writeInt8(writer, 8)
v, ok := row[i].(pql.Decimal)
if !ok {
return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
}
writeInt64(writer, v.ToInt64(v.Scale))
}
case *parser.DataTypeBool:
if val == nil {
writeInt8(writer, 0)
} else {
writeInt8(writer, 1)
v, ok := row[i].(bool)
if !ok {
return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
}
if v {
writeInt8(writer, 1)
} else {
writeInt8(writer, 0)
}
}
case *parser.DataTypeTimestamp:
if val == nil {
writeInt8(writer, 0)
} else {
writeInt8(writer, 8)
v, ok := row[i].(time.Time)
if !ok {
return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
}
writeInt64(writer, v.UnixNano())
}
case *parser.DataTypeIDSet:
if val == nil {
writeInt16(writer, 0)
} else {
v, ok := row[i].([]int64)
if !ok {
return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
}
writeInt16(writer, int16(len(v)))
for _, s := range v {
writeInt64(writer, s)
}
}
case *parser.DataTypeString:
if val == nil {
writeInt16(writer, 0)
} else {
v, ok := row[i].(string)
if !ok {
return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
}
writeInt16(writer, int16(len(v)))
writer.WriteString(v)
}
case *parser.DataTypeStringSet:
if val == nil {
writeInt16(writer, 0)
} else {
v, ok := row[i].([]string)
if !ok {
return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
}
writeInt16(writer, int16(len(v)))
for _, s := range v {
writeInt16(writer, int16(len(s)))
writer.WriteString(s)
}
}
default:
return []byte{}, errors.Errorf("unexpected type '%T'", s.Type)
}
}
writer.Flush()
return buf.Bytes(), nil
}
func ReadRow(reader io.Reader, schema types.Schema) (types.Row, error) {
row := make(types.Row, len(schema))
for idx, s := range schema {
switch t := s.Type.(type) {
case *parser.DataTypeID, *parser.DataTypeInt:
var len int8
err := binary.Read(reader, binary.BigEndian, &len)
if err != nil {
return nil, err
}
if len == 0 {
row[idx] = nil
} else {
var value int64
err := binary.Read(reader, binary.BigEndian, &value)
if err != nil {
return nil, err
}
row[idx] = value
}
case *parser.DataTypeDecimal:
var len int8
err := binary.Read(reader, binary.BigEndian, &len)
if err != nil {
return nil, err
}
if len == 0 {
row[idx] = nil
} else {
var value int64
err := binary.Read(reader, binary.BigEndian, &value)
if err != nil {
return nil, err
}
row[idx] = pql.NewDecimal(value, t.Scale)
}
case *parser.DataTypeBool:
var len int8
err := binary.Read(reader, binary.BigEndian, &len)
if err != nil {
return nil, err
}
if len == 0 {
row[idx] = nil
} else {
var value int8
err := binary.Read(reader, binary.BigEndian, &value)
if err != nil {
return nil, err
}
row[idx] = value == 1
}
case *parser.DataTypeTimestamp:
var len int8
err := binary.Read(reader, binary.BigEndian, &len)
if err != nil {
return nil, err
}
if len == 0 {
row[idx] = nil
} else {
var value int64
err := binary.Read(reader, binary.BigEndian, &value)
if err != nil {
return nil, err
}
row[idx] = time.Unix(0, value)
}
case *parser.DataTypeIDSet:
var len int16
err := binary.Read(reader, binary.BigEndian, &len)
if err != nil {
return nil, err
}
if len == 0 {
row[idx] = nil
} else {
set := make([]int64, len)
for j, _ := range set {
var value int64
err := binary.Read(reader, binary.BigEndian, &value)
if err != nil {
return nil, err
}
set[j] = value
}
row[idx] = set
}
case *parser.DataTypeString:
var len int16
err := binary.Read(reader, binary.BigEndian, &len)
if err != nil {
return nil, err
}
if len == 0 {
row[idx] = nil
} else {
bvalue := make([]byte, len)
err = binary.Read(reader, binary.BigEndian, &bvalue)
if err != nil {
return nil, err
}
row[idx] = string(bvalue)
}
case *parser.DataTypeStringSet:
var len int16
err := binary.Read(reader, binary.BigEndian, &len)
if err != nil {
return nil, err
}
if len == 0 {
row[idx] = nil
} else {
set := make([]string, len)
for j, _ := range set {
var vlen int16
err = binary.Read(reader, binary.BigEndian, &vlen)
if err != nil {
return nil, err
}
bvalue := make([]byte, vlen)
err = binary.Read(reader, binary.BigEndian, &bvalue)
if err != nil {
return nil, err
}
set[j] = string(bvalue)
}
row[idx] = set
}
default:
return nil, errors.Errorf("unexpected type '%T'", s.Type)
}
}
return row, nil
}
// TOKEN_DONE message
// length (bytes)
// token 2
func WriteDone() []byte {
buf := new(bytes.Buffer)
writer := bufio.NewWriter(buf)
// write token
writeToken(writer, TOKEN_DONE)
writer.Flush()
return buf.Bytes()
}
// TOKEN_ERROR_MESSAGE message
// length (bytes)
// token 2
//
// message len 4
// message (from prev)
func WriteError(err error) []byte {
buf := new(bytes.Buffer)
writer := bufio.NewWriter(buf)
// write token
writeToken(writer, TOKEN_ERROR_MESSAGE)
// write error len
t := err.Error()
b := make([]byte, 4)
binary.BigEndian.PutUint32(b, uint32(len(t)))
writer.Write(b)
// write error
writer.WriteString(t)
writer.Flush()
return buf.Bytes()
}
func ReadError(reader io.Reader) (string, error) {
var len int16
err := binary.Read(reader, binary.BigEndian, &len)
if err != nil {
return "", err
}
bvalue := make([]byte, len)
err = binary.Read(reader, binary.BigEndian, &bvalue)
if err != nil {
return "", err
}
return string(bvalue), nil
}
func writeToken(w io.Writer, token int16) {
writeInt16(w, token)
}
func writeInt8(w io.Writer, i int8) {
b := make([]byte, 1)
b[0] = byte(i)
w.Write(b)
}
func writeInt16(w io.Writer, i int16) {
b := make([]byte, 2)
binary.BigEndian.PutUint16(b, uint16(i))
w.Write(b)
}
func writeInt64(w io.Writer, i int64) {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(i))
w.Write(b)
}

144
wireprotocol/wireprimitives_test.go Normal file
View file

@ -0,0 +1,144 @@
package wireprotocol_test
import (
"bytes"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/molecula/featurebase/v3/pql"
"github.com/molecula/featurebase/v3/sql3/parser"
"github.com/molecula/featurebase/v3/sql3/planner/types"
"github.com/molecula/featurebase/v3/wireprotocol"
)
func TestWireProtocol_Schema(t *testing.T) {
s := types.Schema{
&types.PlannerColumn{
ColumnName: "col1",
Type: parser.NewDataTypeID(),
},
&types.PlannerColumn{
ColumnName: "col2",
Type: parser.NewDataTypeInt(),
},
&types.PlannerColumn{
ColumnName: "col3",
Type: parser.NewDataTypeDecimal(4),
},
&types.PlannerColumn{
ColumnName: "col4",
Type: parser.NewDataTypeString(),
},
&types.PlannerColumn{
ColumnName: "col5",
Type: parser.NewDataTypeStringSet(),
},
&types.PlannerColumn{
ColumnName: "col6",
Type: parser.NewDataTypeIDSet(),
},
&types.PlannerColumn{
ColumnName: "col7",
Type: parser.NewDataTypeBool(),
},
&types.PlannerColumn{
ColumnName: "col8",
Type: parser.NewDataTypeTimestamp(),
},
}
b, err := wireprotocol.WriteSchema(s)
if err != nil {
t.Fatal(err)
}
rdr := bytes.NewReader(b)
_, err = wireprotocol.ExpectToken(rdr, wireprotocol.TOKEN_SCHEMA_INFO)
if err != nil {
t.Fatal(err)
}
sr, err := wireprotocol.ReadSchema(rdr)
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(s, sr); diff != "" {
t.Fatal(diff)
}
}
func TestWireProtocol_Row(t *testing.T) {
s := types.Schema{
&types.PlannerColumn{
ColumnName: "col1",
Type: parser.NewDataTypeID(),
},
&types.PlannerColumn{
ColumnName: "col2",
Type: parser.NewDataTypeInt(),
},
&types.PlannerColumn{
ColumnName: "col3",
Type: parser.NewDataTypeDecimal(4),
},
&types.PlannerColumn{
ColumnName: "col4",
Type: parser.NewDataTypeString(),
},
&types.PlannerColumn{
ColumnName: "col5",
Type: parser.NewDataTypeStringSet(),
},
&types.PlannerColumn{
ColumnName: "col6",
Type: parser.NewDataTypeIDSet(),
},
&types.PlannerColumn{
ColumnName: "col7",
Type: parser.NewDataTypeBool(),
},
&types.PlannerColumn{
ColumnName: "col8",
Type: parser.NewDataTypeTimestamp(),
},
}
r := types.Row{
int64(1),
int64(2),
pql.NewDecimal(123400, 4),
string("foo"),
[]string{"bar", "baz"},
[]int64{10, 20},
bool(false),
time.Now().UTC(),
}
b, err := wireprotocol.WriteRow(r, s)
if err != nil {
t.Fatal(err)
}
rdr := bytes.NewReader(b)
_, err = wireprotocol.ExpectToken(rdr, wireprotocol.TOKEN_ROW)
if err != nil {
t.Fatal(err)
}
rr, err := wireprotocol.ReadRow(rdr, s)
if err != nil {
t.Fatal(err)
}
opt := cmp.Comparer(func(x, y pql.Decimal) bool {
return x.EqualTo(y)
})
if diff := cmp.Diff(r, rr, opt); diff != "" {
t.Fatal(diff)
}
}