introduce performance counters and system table fanout, plus refactor metrics (#2363)

* performance counters * first cut of perf counters and system table fanout and a wire protocol * significantly refactored prometheus support; removed statsd and exprvar * removed node_id * put dax subquery test back * Change Translator.TranslateFieldIDs method to take a dax.TableKeyer There are a bunch of other calls to the Translator interface methods with currently take an `index string`, and those need to be converted to dax.TableKeyer as well. But I need to review each call, because in at least one place I noticed one being called with `result.Index` instead of with the qtbl available. And I don't yet know how those could be different. Co-authored-by: Travis Turner <travis@molecula.com> (cherry picked from commit 7f6ea0e6e5)
2026-08-28 02:44:59 +00:00 · 2023-01-11 20:37:00 -06:00 · 2023-01-11 20:37:00 -06:00 · a1fc6d04a1
commit a1fc6d04a1
parent b6d290487d
55 changed files with 3083 additions and 1534 deletions
--- a/4
+++ b/4
@ -20,12 +20,12 @@ RUN apt install -y docker.io
 ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
 RUN chmod +x /usr/local/bin/docker-compose

-WORKDIR /go/src/github.com/molecula/featurebase/cmd/featurebase
+WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase

 # generate an instrumented binary to allow for calculating code coverage for clustertests
 # the entrypoint for the binary is TestRunMain, which is wrapper for main
 RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
-RUN cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
+RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase

 COPY NOTICE /NOTICE

--- a/6
+++ b/6
@ -19,10 +19,10 @@ RUN apt install -y docker.io
 ADD https://github.com/docker/compose/releases/latest/download/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose
 RUN chmod +x /usr/local/bin/docker-compose

-WORKDIR /go/src/github.com/molecula/featurebase/cmd/featurebase
+WORKDIR /go/src/github.com/featurebasedb/featurebase/cmd/featurebase

 RUN go test -covermode=atomic -coverpkg=../../... -c -tags testrunmain -o featurebase
-RUN cp /go/src/github.com/molecula/featurebase/cmd/featurebase/featurebase /featurebase
+RUN cp /go/src/github.com/featurebasedb/featurebase/cmd/featurebase/featurebase /featurebase


 COPY NOTICE /NOTICE
@ -32,6 +32,6 @@ COPY ./internal/clustertests /go/src/github.com/featurebasedb/featurebase/intern
 EXPOSE 10101
 VOLUME /data

-WORKDIR /go/src/github.com/molecula/featurebase
+WORKDIR /go/src/github.com/featurebasedb/featurebase

 CMD ["/featurebase", "-test.run=TestRunMain", "-test.coverprofile=/results/coverage.out", "server", "--data-dir", "/data", "--bind", "http://0.0.0.0:10101"]
--- a/api.go
+++ b/api.go
@ -278,7 +278,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
 		return nil, errors.Wrap(err, "creating index")
 	}

-	api.holder.Stats.Count(MetricCreateIndex, 1, 1.0)
+	CounterCreateIndex.Inc()
 	return index, nil
 }

@ -319,7 +319,7 @@ func (api *API) DeleteDataframe(ctx context.Context, indexName string) error {
 		api.server.logger.Errorf("problem sending DeleteIndex message: %s", err)
 		return errors.Wrap(err, "sending DeleteIndex message")
 	}
-	api.holder.Stats.Count(MetricDeleteDataframe, 1, 1.0)
+	CounterDeleteDataframe.Inc()
 	return nil
 }

@ -354,7 +354,7 @@ func (api *API) DeleteIndex(ctx context.Context, indexName string) error {
 			return errors.Wrap(err, "deleting id allocation for index")
 		}
 	}
-	api.holder.Stats.Count(MetricDeleteIndex, 1, 1.0)
+	CounterDeleteIndex.Inc()
 	return nil
 }

@ -406,7 +406,7 @@ func (api *API) CreateField(ctx context.Context, indexName string, fieldName str
 		return nil, errors.Wrap(err, "sending CreateField message")
 	}

-	api.holder.Stats.CountWithCustomTags(MetricCreateField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
+	CounterCreateField.With(prometheus.Labels{"index": indexName})
 	return field, nil
 }

@ -750,7 +750,7 @@ func (api *API) DeleteField(ctx context.Context, indexName string, fieldName str
 		api.server.logger.Errorf("problem sending DeleteField message: %s", err)
 		return errors.Wrap(err, "sending DeleteField message")
 	}
-	api.holder.Stats.CountWithCustomTags(MetricDeleteField, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
+	CounterDeleteField.With(prometheus.Labels{"index": indexName})
 	return nil
 }

@ -782,7 +782,7 @@ func (api *API) DeleteAvailableShard(_ context.Context, indexName, fieldName str
 		api.server.logger.Errorf("problem sending DeleteAvailableShard message: %s", err)
 		return errors.Wrap(err, "sending DeleteAvailableShard message")
 	}
-	api.holder.Stats.CountWithCustomTags(MetricDeleteAvailableShard, 1, 1.0, []string{fmt.Sprintf("index:%s", indexName)})
+	CounterDeleteAvailableShard.With(prometheus.Labels{"index": indexName}).Inc()
 	return nil
 }

@ -2096,15 +2096,6 @@ func (api *API) AvailableShards(ctx context.Context, indexName string) (*roaring
 	return index.AvailableShards(false), nil
 }

-// StatsWithTags returns an instance of whatever implementation of StatsClient
-// pilosa is using with the given tags.
-func (api *API) StatsWithTags(tags []string) stats.StatsClient {
-	if api.holder == nil || api.cluster == nil {
-		return nil
-	}
-	return api.holder.Stats.WithTags(tags...)
-}
-
 // LongQueryTime returns the configured threshold for logging/statting
 // long running queries.
 func (api *API) LongQueryTime() time.Duration {
@ -2391,19 +2382,19 @@ func (api *API) StartTransaction(ctx context.Context, id string, timeout time.Du
 	switch err {
 	case nil:
 		if exclusive {
-			api.holder.Stats.Count(MetricExclusiveTransactionRequest, 1, 1.0)
+			CounterExclusiveTransactionRequest.Inc()
 		} else {
-			api.holder.Stats.Count(MetricTransactionStart, 1, 1.0)
+			CounterTransactionStart.Inc()
 		}
 	case ErrTransactionExclusive:
 		if exclusive {
-			api.holder.Stats.Count(MetricExclusiveTransactionBlocked, 1, 1.0)
+			CounterExclusiveTransactionBlocked.Inc()
 		} else {
-			api.holder.Stats.Count(MetricTransactionBlocked, 1, 1.0)
+			CounterTransactionBlocked.Inc()
 		}
 	}
 	if exclusive && t != nil && t.Active {
-		api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
+		CounterExclusiveTransactionActive.Inc()
 	}
 	return t, err
 }
@ -2415,9 +2406,9 @@ func (api *API) FinishTransaction(ctx context.Context, id string, remote bool) (
 	t, err := api.server.FinishTransaction(ctx, id, remote)
 	if err == nil {
 		if t.Exclusive {
-			api.holder.Stats.Count(MetricExclusiveTransactionEnd, 1, 1.0)
+			CounterExclusiveTransactionEnd.Inc()
 		} else {
-			api.holder.Stats.Count(MetricTransactionEnd, 1, 1.0)
+			CounterTransactionEnd.Inc()
 		}
 	}
 	return t, err
@ -2437,7 +2428,7 @@ func (api *API) GetTransaction(ctx context.Context, id string, remote bool) (*Tr
 	t, err := api.server.GetTransaction(ctx, id, remote)
 	if err == nil {
 		if t.Exclusive && t.Active {
-			api.holder.Stats.Count(MetricExclusiveTransactionActive, 1, 1.0)
+			CounterExclusiveTransactionActive.Inc()
 		}
 	}
 	return t, err
@ -3046,6 +3037,10 @@ func (api *API) CompilePlan(ctx context.Context, q string) (planner_types.PlanOp
 	return api.server.CompileExecutionPlan(ctx, q)
 }

+func (api *API) RehydratePlanOperator(ctx context.Context, reader io.Reader) (planner_types.PlanOperator, error) {
+	return api.server.RehydratePlanOperator(ctx, reader)
+}
+
 func (api *API) RBFDebugInfo() map[string]*rbf.DebugInfo {
 	infos := make(map[string]*rbf.DebugInfo)

@ -3343,6 +3338,7 @@ type SystemAPI interface {
 	ClusterState() string
 	DataDir() string

+	NodeID() string
 	ClusterNodes() []ClusterNode
 }

@ -3414,6 +3410,10 @@ func (fsapi *FeatureBaseSystemAPI) DataDir() string {
 	return fsapi.server.dataDir
 }

+func (fsapi *FeatureBaseSystemAPI) NodeID() string {
+	return fsapi.cluster.Node.ID
+}
+
 func (fsapi *FeatureBaseSystemAPI) ClusterNodes() []ClusterNode {
 	result := make([]ClusterNode, 0)

--- a/batch/Dockerfile-test
+++ b/batch/Dockerfile-test
@ -2,10 +2,10 @@ ARG GO_VERSION=1.19

 FROM golang:${GO_VERSION}

-WORKDIR /go/src/github.com/molecula/featurebase/
+WORKDIR /go/src/github.com/featurebasedb/featurebase/

 COPY . .

-WORKDIR /go/src/github.com/molecula/featurebase/batch/
+WORKDIR /go/src/github.com/featurebasedb/featurebase/batch/

 CMD ["go","test","-v","-mod=vendor","-tags=odbc,dynamic","./..."]
--- a/batch/batch.go
+++ b/batch/batch.go
@ -755,7 +755,7 @@ func (b *Batch) Import() error {
 		}()
 	}
 	defer func() {
-		b.importer.StatsTiming(MetricBatchImportDurationSeconds, time.Since(start), 1.0)
+		featurebase.SummaryBatchImportDurationSeconds.Observe(time.Since(start).Seconds())
 	}()

 	size := len(b.ids)
@ -828,7 +828,7 @@ func (b *Batch) Flush() error {
 		if err != nil {
 			b.log.Errorf("error finishing transaction: %v. trns: %+v", err, trnsl)
 		}
-		b.importer.StatsTiming(MetricBatchFlushDurationSeconds, time.Since(start), 1.0)
+		featurebase.SummaryBatchFlushDurationSeconds.Observe(time.Since(start).Seconds())
 	}()

 	importStart := time.Now()
@ -1188,7 +1188,7 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error {
 		}
 	}

-	b.importer.StatsTiming(MetricBatchShardImportBuildRequestsSeconds, time.Since(start), 1.0)
+	featurebase.SummaryBatchShardImportBuildRequestsSeconds.Observe(time.Since(start).Seconds())
 	start = time.Now()
 	eg := egpool.Group{PoolSize: 20}
 	for shard, request := range requests {
@ -1200,7 +1200,7 @@ func (b *Batch) doImportShardTransactional(frags, clearFrags fragments) error {
 	}
 	err := eg.Wait()
 	dur := time.Since(start)
-	b.importer.StatsTiming(MetricBatchShardImportDurationSeconds, dur, 1.0)
+	featurebase.SummaryBatchImportDurationSeconds.Observe(dur.Seconds())
 	b.log.Printf("import shard took: %v\n", dur)
 	return errors.Wrap(err, "doing shard-transactional imports")
 }
--- a/batch/metrics.go
+++ b/batch/metrics.go
@ -1,28 +1,3 @@
 // Copyright 2022 Molecula Corp. (DBA FeatureBase).
 // SPDX-License-Identifier: Apache-2.0
 package batch
-
-const (
-	// MetricBatchImportDurationSeconds records the full time of the
-	// RecordBatch.Import call. This includes starting and finishing a
-	// transaction, doing key translation, building fragments locally,
-	// importing all data, and resetting internal structures.
-	MetricBatchImportDurationSeconds = "batch_import_duration_seconds"
-
-	// MetricBatchFlushDurationSeconds records the full time for
-	// RecordBatch.Flush (if splitBatchMode is in use). This includes
-	// starting and finishing a transaction, importing all data, and
-	// resetting internal structures.
-	MetricBatchFlushDurationSeconds = "batch_flush_duration_seconds"
-
-	// MetricBatchShardImportBuildRequestsSeconds is the time it takes
-	// after making fragments to build the shard-transactional request
-	// objects (but not actually import them or do any network activity).
-	MetricBatchShardImportBuildRequestsSeconds = "batch_shard_import_build_requests_seconds"
-
-	// MetricBatchShardImportDurationSeconds is the time it takes to
-	// import all data for all shards in the batch using the
-	// shard-transactional endpoint. This does not include the time it
-	// takes to build the requests locally.
-	MetricBatchShardImportDurationSeconds = "batch_shard_import_duration_seconds"
-)
--- a/cache.go
+++ b/cache.go
@ -41,9 +41,6 @@ type cache interface {
 	// Returns an ordered list of the top ranked bitmaps.
 	Top() []bitmapPair

-	// SetStats defines the stats client used in the cache.
-	SetStats(s stats.StatsClient)
-
 	// Clear removes everything from the cache. If possible it should leave allocated structures in place to be reused.
 	Clear()
 }
@ -52,7 +49,6 @@ type cache interface {
 type lruCache struct {
 	cache  *lru.Cache
 	counts map[uint64]uint64
-	stats  stats.StatsClient
 	// maxEntries is saved to support Clear which recreates the cache.
 	maxEntries uint32
 }
@ -62,7 +58,6 @@ func newLRUCache(maxEntries uint32) *lruCache {
 	c := &lruCache{
 		cache:      lru.New(int(maxEntries)),
 		counts:     make(map[uint64]uint64),
-		stats:      stats.NopStatsClient,
 		maxEntries: maxEntries,
 	}
 	c.cache.OnEvicted = c.onEvicted
@ -120,11 +115,6 @@ func (c *lruCache) Top() []bitmapPair {
 	return a
 }

-// SetStats defines the stats client used in the cache.
-func (c *lruCache) SetStats(s stats.StatsClient) {
-	c.stats = s
-}
-
 func (c *lruCache) Clear() {
 	for k := range c.counts {
 		delete(c.counts, k)
@ -158,8 +148,6 @@ type rankCache struct {

 	// thresholdValue is the value of the last item in the cache
 	thresholdValue uint64
-
-	stats stats.StatsClient
 }

 // NewRankCache returns a new instance of RankCache.
@ -168,7 +156,6 @@ func NewRankCache(maxEntries uint32) *rankCache {
 		maxEntries:      maxEntries,
 		thresholdBuffer: int(thresholdFactor * float64(maxEntries)),
 		entries:         make(map[uint64]uint64),
-		stats:           stats.NopStatsClient,
 	}
 }

@ -229,7 +216,7 @@ func (c *rankCache) BulkAdd(id uint64, n uint64) {
 	// as this can take up an upbounded amount of memory. This is especially
 	// true when restoring shards as all rows will be added.
 	if len(c.entries) > int(2*c.maxEntries) {
-		c.stats.Count(MetricRecalculateCache, 1, 1.0)
+		CounterRecalculateCache.Inc()
 		c.recalculate()
 	}
 }
@ -274,7 +261,7 @@ func (c *rankCache) Invalidate() {
 func (c *rankCache) Recalculate() {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	c.stats.Count(MetricRecalculateCache, 1, 1.0)
+	CounterRecalculateCache.Inc()
 	c.recalculate()
 }

@ -286,12 +273,12 @@ func (c *rankCache) invalidate() {
 		// This is somewhat necessary for now since recalculation is not cheap.
 		// The cache will remain flagged as dirty and will be recalculated if Top is called.
 		// This may cause unexpected memory growth, so record it in metrics for debugging purposes.
-		c.stats.Count(MetricInvalidateCacheSkipped, 1, 1.0)
+		CounterInvalidateCacheSkipped.Inc()
 		// Ensure that we're marked as dirty even if we weren't otherwise.
 		c.dirty = true
 		return
 	}
-	c.stats.Count(MetricInvalidateCache, 1, 1.0)
+	CounterInvalidateCache.Inc()
 	c.recalculate()
 }

@ -317,7 +304,7 @@ func (c *rankCache) recalculate() {

 	// Store the count of the item at the threshold index.
 	length := len(c.rankings)
-	c.stats.Gauge(MetricRankCacheLength, float64(length), 1.0)
+	GaugeRankCacheLength.Set(float64(length))

 	var removeItems []bitmapPair // cached, ordered list
 	if length > int(c.maxEntries) {
@ -333,7 +320,7 @@ func (c *rankCache) recalculate() {

 	// If size is larger than the threshold then trim it.
 	if len(c.entries) > c.thresholdBuffer {
-		c.stats.Count(MetricCacheThresholdReached, 1, 1.0)
+		CounterCacheThresholdReached.Inc()
 		for _, pair := range removeItems {
 			delete(c.entries, pair.ID)
 		}
@ -343,11 +330,6 @@ func (c *rankCache) recalculate() {
 	c.dirty = false
 }

-// SetStats defines the stats client used in the cache.
-func (c *rankCache) SetStats(s stats.StatsClient) {
-	c.stats = s
-}
-
 // Top returns an ordered list of pairs.
 func (c *rankCache) Top() []bitmapPair {
 	c.mu.Lock()
@ -355,7 +337,7 @@ func (c *rankCache) Top() []bitmapPair {

 	if c.dirty {
 		// The cache is dirty, so we need to recalculate it to get a consistent view.
-		c.stats.Count(MetricReadDirtyCache, 1, 1.0)
+		CounterReadDirtyCache.Inc()
 		c.recalculate()
 	}

@ -606,25 +588,21 @@ func (p uint64Slice) Len() int           { return len(p) }
 func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] }

 // nopCache represents a no-op Cache implementation.
-type nopCache struct {
-	stats stats.StatsClient
-}
+type nopCache struct{}

 // Ensure NopCache implements Cache.
-var globalNopCache cache = nopCache{
-	stats: stats.NopStatsClient,
-}
+var globalNopCache cache = nopCache{}

 func (c nopCache) Add(uint64, uint64)     {}
 func (c nopCache) BulkAdd(uint64, uint64) {}
 func (c nopCache) Get(uint64) uint64      { return 0 }
 func (c nopCache) IDs() []uint64          { return []uint64{} }

-func (c nopCache) Invalidate()                {}
-func (c nopCache) Len() int                   { return 0 }
-func (c nopCache) Recalculate()               {}
-func (c nopCache) SetStats(stats.StatsClient) {}
-func (c nopCache) Clear()                     {}
+func (c nopCache) Invalidate()  {}
+func (c nopCache) Len() int     { return 0 }
+func (c nopCache) Recalculate() {}
+
+func (c nopCache) Clear() {}

 func (c nopCache) Top() []bitmapPair {
 	return []bitmapPair{}
--- a/client/client.go
+++ b/client/client.go
@ -52,7 +52,6 @@ type Client struct {
 	manualFragmentNode *fragmentNode
 	manualServerURI    *pnet.URI
 	tracer             opentracing.Tracer
-	Stats              stats.StatsClient
 	// An exponential backoff algorithm retries requests exponentially (if an HTTP request fails),
 	// increasing the waiting time between retries up to a maximum backoff time.
 	maxBackoff time.Duration
@ -211,11 +210,6 @@ func newClientWithOptions(options *ClientOptions) *Client {
 	} else {
 		c.tracer = options.tracer
 	}
-	if options.stats == nil {
-		c.Stats = stats.NopStatsClient
-	} else {
-		c.Stats = options.stats
-	}

 	c.maxRetries = *options.retries
 	c.maxBackoff = 2 * time.Minute
@ -1359,7 +1353,6 @@ type ClientOptions struct {
 	manualServerAddress bool
 	tracer              opentracing.Tracer
 	retries             *int
-	stats               stats.StatsClient
 	nat                 map[pnet.URI]pnet.URI
 	pathPrefix          string
 }
@ -1445,14 +1438,6 @@ func OptClientRetries(retries int) ClientOption {
 	}
 }

-// OptClientStatsClient sets a stats client, such as Prometheus
-func OptClientStatsClient(stats stats.StatsClient) ClientOption {
-	return func(options *ClientOptions) error {
-		options.stats = stats
-		return nil
-	}
-}
-
 // OptClientNAT sets a NAT map used to translate the advertised URI to something
 // else (for example, when accessing pilosa running in docker).
 func OptClientNAT(nat map[string]string) ClientOption {
--- a/client/importer.go
+++ b/client/importer.go
@ -328,7 +328,3 @@ func (i *importer) EncodeImport(ctx context.Context, tid dax.TableID, fld *dax.F
 func (i *importer) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, path string, data []byte) error {
 	return i.client.DoImport(string(tid), shard, path, data)
 }
-
-func (i *importer) StatsTiming(name string, value time.Duration, rate float64) {
-	i.client.Stats.Timing(name, value, rate)
-}
--- a/dax/README.md
+++ b/dax/README.md
@ -10,7 +10,7 @@ the Controller, as well as FeatureBase and IDK-based ingesters.
 The DAX test currently requires docker images for: `featurebase`  and `datagen`.

 If at any point you run into problems with go mod failing to reference a private
-repo, make sure that you have `gitlab.com/molecula` in your `GOPRIVATE`
+repo, make sure that you have `gitlab.com/featurebasedb` in your `GOPRIVATE`
 environment variable.

 Note that during the docker image build step, `go mod vendor` is run, which
@ -24,7 +24,7 @@ These may no longer be relevant.

 I needed to but this in my `~/.profile` file:

-```export GOPRIVATE=github.com/molecula,gitlab.com/molecula```
+```export GOPRIVATE=github.com/featurebasedb,gitlab.com/featurebasedb```

 And this in my `~/.gitconfig`

@ -41,9 +41,9 @@ Then `make docker` ran successfully.
 ### Build the FeatureBase docker image

 - Check out the
-  [dax](https://github.com/molecula/featurebase/tree/dax)
+  [dax](https://github.com/featurebasedb/featurebase/tree/dax)
  branch of the
-  [featurebase](https://github.com/molecula/featurebase) repository.
+  [featurebase](https://github.com/featurebasedb/featurebase) repository.
 - Run `make docker-image-featurebase` to build the docker image
 - You should now have an image in docker named `dax/featurebase` with the tag `latest`.

@ -57,8 +57,8 @@ Then `make docker` ran successfully.
 ## Running the tests

 - Check out the
-  [dax](https://github.com/molecula/featurebase/tree/dax)
+  [dax](https://github.com/featurebasedb/featurebase/tree/dax)
  branch of the
-  [featurebase](https://github.com/molecula/featurebase) repository.
+  [featurebase](https://github.com/featurebasedb/featurebase) repository.
 - Change into the `dax` directory: `cd dax`
 - Run `make test-integration`.
--- a/dax/queryer/orchestrator.go
+++ b/dax/queryer/orchestrator.go
@ -74,7 +74,7 @@ type Translator interface {
 	// TODO(jaffee) the naming here is a cluster. TranslateIndexIDs takes a list, but TranslateFieldIDs takes a set, both have alternate methods that take the other thing. :facepalm:
 	TranslateIndexIDs(ctx context.Context, index string, ids []uint64) ([]string, error)
 	TranslateIndexIDSet(ctx context.Context, index string, ids map[uint64]struct{}) (map[uint64]string, error)
-	TranslateFieldIDs(ctx context.Context, index, field string, ids map[uint64]struct{}) (map[uint64]string, error)
+	TranslateFieldIDs(ctx context.Context, tableKeyer dax.TableKeyer, field string, ids map[uint64]struct{}) (map[uint64]string, error)
 	TranslateFieldListIDs(ctx context.Context, index, field string, ids []uint64) ([]string, error)
 }

@ -87,7 +87,6 @@ type orchestrator struct {
 	// Client used for remote requests.
 	client *featurebase.InternalClient

-	stats  stats.StatsClient
 	logger logger.Logger
 }

@ -433,11 +432,11 @@ func (o *orchestrator) executeCall(ctx context.Context, tableKeyer dax.TableKeye
 	} else if err := o.validateCallArgs(c); err != nil {
 		return nil, errors.Wrap(err, "validating args")
 	}
-	indexTag := "index:" + string(tableKeyer.Key())
-	metricName := "query_" + strings.ToLower(c.Name) + "_total"
-	statFn := func() {
+
+	labels := prometheus.Labels{"index": string(tableKeyer.Key())}
+	statFn := func(ctr *prometheus.CounterVec) {
 		if !opt.Remote {
-			o.stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag})
+			ctr.With(labels).Inc()
 		}
 	}

@ -449,101 +448,106 @@ func (o *orchestrator) executeCall(ctx context.Context, tableKeyer dax.TableKeye

 	switch c.Name {
 	case "Sum":
-		statFn()
+		statFn(featurebase.CounterQuerySumTotal)
 		res, err := o.executeSum(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeSum")
 	case "Min":
-		statFn()
+		statFn(featurebase.CounterQueryMinTotal)
 		res, err := o.executeMin(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeMin")
 	case "Max":
-		statFn()
+		statFn(featurebase.CounterQueryMaxTotal)
 		res, err := o.executeMax(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeMax")
 	case "MinRow":
-		statFn()
+		statFn(featurebase.CounterQueryMinRowTotal)
 		res, err := o.executeMinRow(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeMinRow")
 	case "MaxRow":
-		statFn()
+		statFn(featurebase.CounterQueryMaxRowTotal)
 		res, err := o.executeMaxRow(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeMaxRow")
 	// case "Clear":
-	// 	statFn()
+	// 	statFn(featurebase.CounterQueryClearTotal)
 	// 	res, err := o.executeClearBit(ctx, index, c, opt)
 	// 	return res, errors.Wrap(err, "executeClearBit")
 	// case "ClearRow":
-	// 	statFn()
+	// 	statFn(featurebase.CounterQueryClearRowTotal)
 	// 	res, err := o.executeClearRow(ctx, index, c, shards, opt)
 	// 	return res, errors.Wrap(err, "executeClearRow")
 	case "Distinct":
-		statFn()
+		statFn(featurebase.CounterQueryDistinctTotal)
 		res, err := o.executeDistinct(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeDistinct")
 	// case "Store":
-	// 	statFn()
+	// 	statFn(featurebase.CounterQueryStoreTotal)
 	// 	res, err := o.executeSetRow(ctx, index, c, shards, opt)
 	// 	return res, errors.Wrap(err, "executeSetRow")
 	case "Count":
-		statFn()
+		statFn(featurebase.CounterQueryCountTotal)
 		res, err := o.executeCount(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeCount")
 	// case "Set":
-	// 	statFn()
+	// 	statFn(featurebase.CounterQuerySetTotal)
 	// 	res, err := o.executeSet(ctx, index, c, opt)
 	// 	return res, errors.Wrap(err, "executeSet")
 	case "TopK":
-		statFn()
+		statFn(featurebase.CounterQueryTopKTotal)
 		res, err := o.executeTopK(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeTopK")
 	case "TopN":
-		statFn()
+		statFn(featurebase.CounterQueryTopNTotal)
 		res, err := o.executeTopN(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeTopN")
 	case "Rows":
-		statFn()
+		statFn(featurebase.CounterQueryRowsTotal)
 		res, err := o.executeRows(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeRows")
 	case "Extract":
-		statFn()
+		statFn(featurebase.CounterQueryExtractTotal)
 		res, err := o.executeExtract(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeExtract")
 	case "GroupBy":
-		statFn()
+		statFn(featurebase.CounterQueryGroupByTotal)
 		res, err := o.executeGroupBy(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeGroupBy")
 	case "Options":
-		statFn()
+		statFn(featurebase.CounterQueryOptionsTotal)
 		res, err := o.executeOptionsCall(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeOptionsCall")
 	case "IncludesColumn":
+		statFn(featurebase.CounterQueryIncludesColumnTotal)
 		res, err := o.executeIncludesColumnCall(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeIncludesColumnCall")
 	case "FieldValue":
-		statFn()
+		statFn(featurebase.CounterQueryFieldValueTotal)
 		res, err := o.executeFieldValueCall(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeFieldValueCall")
 	case "Precomputed":
+		statFn(featurebase.CounterQueryPrecomputedTotal)
 		res, err := o.executePrecomputedCall(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executePrecomputedCall")
 	case "UnionRows":
+		statFn(featurebase.CounterQueryUnionRowsTotal)
 		res, err := o.executeUnionRows(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeUnionRows")
 	case "ConstRow":
+		statFn(featurebase.CounterQueryConstRowTotal)
 		res, err := o.executeConstRow(ctx, tableKeyer, c)
 		return res, errors.Wrap(err, "executeConstRow")
 	case "Limit":
+		statFn(featurebase.CounterQueryLimitTotal)
 		res, err := o.executeLimitCall(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeLimitCall")
 	case "Percentile":
+		statFn(featurebase.CounterQueryPercentileTotal)
 		res, err := o.executePercentile(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executePercentile")
 	// case "Delete":
-	// 	statFn() //TODO(twg) need this?
+	// 	statFn(featurebase.CounterQueryDeleteTotal)
 	// 	res, err := o.executeDeleteRecords(ctx, index, c, shards, opt)
 	// 	return res, errors.Wrap(err, "executeDelete")
 	default: // o.g. "Row", "Union", "Intersect" or anything that returns a bitmap.
-		statFn()
 		res, err := o.executeBitmapCall(ctx, tableKeyer, c, shards, opt)
 		return res, errors.Wrap(err, "executeBitmapCall")
 	}
@ -1116,13 +1120,42 @@ func (o *orchestrator) executeBitmapCall(ctx context.Context, tableKeyer dax.Tab
 	span.LogKV("pqlCallName", c.Name)
 	defer span.Finish()

-	indexTag := "index:" + string(tableKeyer.Key())
-	metricName := "query_" + strings.ToLower(c.Name) + "_total"
-	if c.Name == "Row" && c.HasConditionArg() {
-		metricName = "query_row_bsi_total"
+	labels := prometheus.Labels{"index": string(tableKeyer.Key())}
+	statFn := func(ctr *prometheus.CounterVec) {
+		if !opt.Remote {
+			ctr.With(labels).Inc()
+		}
 	}
+
 	if !opt.Remote {
-		o.stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag})
+		switch c.Name {
+		case "Row":
+			if c.HasConditionArg() {
+				statFn(featurebase.CounterQueryRowBSITotal)
+			} else {
+				statFn(featurebase.CounterQueryRowTotal)
+			}
+		case "Range":
+			statFn(featurebase.CounterQueryRangeTotal)
+		case "Difference":
+			statFn(featurebase.CounterQueryBitmapTotal)
+		case "Intersect":
+			statFn(featurebase.CounterQueryIntersectTotal)
+		case "Union":
+			statFn(featurebase.CounterQueryUnionTotal)
+		case "InnerUnionRows":
+			statFn(featurebase.CounterQueryInnerUnionRowsTotal)
+		case "Xor":
+			statFn(featurebase.CounterQueryXorTotal)
+		case "Not":
+			statFn(featurebase.CounterQueryNotTotal)
+		case "Shift":
+			statFn(featurebase.CounterQueryShiftTotal)
+		case "All":
+			statFn(featurebase.CounterQueryAllTotal)
+		default:
+			statFn(featurebase.CounterQueryBitmapTotal)
+		}
 	}

 	// Merge returned results at coordinating node.
@ -2985,9 +3018,7 @@ func (o *orchestrator) preTranslateMatrixSet(ctx context.Context, mat featurebas
 		}
 	}

-	index := string(tableKeyer.Key())
-
-	return o.trans.TranslateFieldIDs(ctx, index, field, ids)
+	return o.trans.TranslateFieldIDs(ctx, tableKeyer, field, ids)
 }

 func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedTable, call *pql.Call, result interface{}, idSet map[uint64]string) (_ interface{}, err error) {
@ -3164,7 +3195,7 @@ func (o *orchestrator) translateResult(ctx context.Context, qtbl *dax.QualifiedT

 		fieldTranslations := make(map[string]map[uint64]string)
 		for field, ids := range fieldIDs {
-			trans, err := o.trans.TranslateFieldIDs(ctx, idx.Name, field.Name, ids)
+			trans, err := o.trans.TranslateFieldIDs(ctx, qtbl, field.Name, ids)
 			if err != nil {
 				return nil, errors.Wrapf(err, "translating IDs in field '%q'", field.Name)
 			}
--- a/dax/queryer/queryer.go
+++ b/dax/queryer/queryer.go
@ -89,7 +89,6 @@ func (q *Queryer) Orchestrator(qual dax.TableQualifier) *qualifiedOrchestrator {
 		topology: &MDSTopology{noder: q.noder},
 		// TODO(jaffee) using default http.Client probably bad... need to set some timeouts.
 		client: q.fbClient,
-		stats:  stats.NopStatsClient,
 		logger: q.logger,
 	}

--- a/dax/queryer/translator.go
+++ b/dax/queryer/translator.go
@ -229,15 +229,19 @@ func (m *mdsTranslator) TranslateIndexIDSet(ctx context.Context, table string, i
 	}
 	return ret, nil
 }
-func (m *mdsTranslator) TranslateFieldIDs(ctx context.Context, table, field string, ids map[uint64]struct{}) (map[uint64]string, error) {
+func (m *mdsTranslator) TranslateFieldIDs(ctx context.Context, tableKeyer dax.TableKeyer, field string, ids map[uint64]struct{}) (map[uint64]string, error) {
 	idList := make([]uint64, 0, len(ids))
 	for id := range ids {
 		idList = append(idList, id)
 	}

-	stringList, err := m.TranslateFieldListIDs(ctx, table, field, idList)
+	// TODO(tlt): convert TranslateFieldListIDs (and the other Translator
+	// interface methods) to TableKeyer.
+	index := string(tableKeyer.Key())
+
+	stringList, err := m.TranslateFieldListIDs(ctx, index, field, idList)
 	if err != nil {
-		return nil, errors.Wrapf(err, "translating field ids on field: %s, %s", table, field)
+		return nil, errors.Wrapf(err, "translating field ids on field: %s, %s", tableKeyer, field)
 	}

 	ret := make(map[uint64]string)
--- a/dax/test/dax/dax_test.go
+++ b/dax/test/dax/dax_test.go
@ -112,7 +112,6 @@ func TestDAXIntegration(t *testing.T) {
 			"alterTable/alterTableBadTable", // looks like table does not exist is a different error in DAX
 			"top-tests/test-1",              // don't know why this is failing at all
 			"delete_tests",
-			"subquerytable", // subqueries seem to be a problem
 		}

 		doSkip := func(name string) bool {
--- a/executor.go
+++ b/executor.go
@ -29,6 +29,7 @@ import (
 	"github.com/featurebasedb/featurebase/v3/testhook"
 	"github.com/featurebasedb/featurebase/v3/tracing"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/sync/errgroup"
 )

@ -165,16 +166,16 @@ func (e *executor) Close() error {
 // PoolSize is exported to let the task pool update us
 func (e *executor) PoolSize(n int) {
 	if e.Holder != nil {
-		e.Holder.Stats.Gauge("worker_total", float64(n), 0)
+		GaugeWorkerTotal.Set(float64(n))
 	}
 }

 // InitStats initializes stats counters. Must be called after Holder set.
 func (e *executor) InitStats() {
 	if e.Holder != nil {
-		e.Holder.Stats.Count("job_total", 0, 0)
+		CounterJobTotal.Add(0)
 		l, _, _ := e.workers.Stats()
-		e.Holder.Stats.Gauge("worker_total", float64(l), 0)
+		GaugeWorkerTotal.Set(float64(l))
 	}
 }

@ -684,11 +685,11 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p
 	} else if err := e.validateCallArgs(c); err != nil {
 		return nil, errors.Wrap(err, "validating args")
 	}
-	indexTag := "index:" + index
-	metricName := "query_" + strings.ToLower(c.Name) + "_total"
-	statFn := func() {
+
+	labels := prometheus.Labels{"index": index}
+	statFn := func(ctr *prometheus.CounterVec) {
 		if !opt.Remote {
-			e.Holder.Stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag})
+			ctr.With(labels).Inc()
 		}
 	}

@ -720,114 +721,122 @@ func (e *executor) executeCall(ctx context.Context, qcx *Qcx, index string, c *p

 	switch c.Name {
 	case "Sum":
-		statFn()
+		statFn(CounterQuerySumTotal)
 		res, err := e.executeSum(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeSum")
 	case "Min":
-		statFn()
+		statFn(CounterQueryMinTotal)
 		res, err := e.executeMin(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeMin")
 	case "Max":
-		statFn()
+		statFn(CounterQueryMaxTotal)
 		res, err := e.executeMax(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeMax")
 	case "MinRow":
-		statFn()
+		statFn(CounterQueryMinRowTotal)
 		res, err := e.executeMinRow(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeMinRow")
 	case "MaxRow":
-		statFn()
+		statFn(CounterQueryMaxRowTotal)
 		res, err := e.executeMaxRow(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeMaxRow")
 	case "Clear":
-		statFn()
+		statFn(CounterQueryClearTotal)
 		res, err := e.executeClearBit(ctx, qcx, index, c, opt)
 		return res, errors.Wrap(err, "executeClearBit")
 	case "ClearRow":
-		statFn()
+		statFn(CounterQueryClearRowTotal)
 		res, err := e.executeClearRow(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeClearRow")
 	case "Distinct":
-		statFn()
+		statFn(CounterQueryDistinctTotal)
 		res, err := e.executeDistinct(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeDistinct")
 	case "Store":
-		statFn()
+		statFn(CounterQueryStoreTotal)
 		res, err := e.executeSetRow(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeSetRow")
 	case "Count":
-		statFn()
+		statFn(CounterQueryCountTotal)
 		res, err := e.executeCount(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeCount")
 	case "Set":
-		statFn()
+		statFn(CounterQuerySetTotal)
 		res, err := e.executeSet(ctx, qcx, index, c, opt)
 		return res, errors.Wrap(err, "executeSet")
 	case "TopK":
-		statFn()
+		statFn(CounterQueryTopKTotal)
 		res, err := e.executeTopK(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeTopK")
 	case "TopN":
-		statFn()
+		statFn(CounterQueryTopNTotal)
 		res, err := e.executeTopN(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeTopN")
 	case "Rows":
-		statFn()
+		statFn(CounterQueryRowsTotal)
 		res, err := e.executeRows(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeRows")
 	case "ExternalLookup":
-		statFn()
+		statFn(CounterQueryExternalLookupTotal)
 		res, err := e.executeExternalLookup(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeExternalLookup")
 	case "Extract":
-		statFn()
+		statFn(CounterQueryExtractTotal)
 		res, err := e.executeExtract(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeExtract")
 	case "GroupBy":
-		statFn()
+		statFn(CounterQueryGroupByTotal)
 		res, err := e.executeGroupBy(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeGroupBy")
 	case "Options":
-		statFn()
+		statFn(CounterQueryOptionsTotal)
 		res, err := e.executeOptionsCall(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeOptionsCall")
 	case "IncludesColumn":
+		statFn(CounterQueryIncludesColumnTotal)
 		res, err := e.executeIncludesColumnCall(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeIncludesColumnCall")
 	case "FieldValue":
-		statFn()
+		statFn(CounterQueryFieldValueTotal)
 		res, err := e.executeFieldValueCall(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeFieldValueCall")
 	case "Precomputed":
+		statFn(CounterQueryPrecomputedTotal)
 		res, err := e.executePrecomputedCall(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executePrecomputedCall")
 	case "UnionRows":
+		statFn(CounterQueryUnionRowsTotal)
 		res, err := e.executeUnionRows(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeUnionRows")
 	case "ConstRow":
+		statFn(CounterQueryConstRowTotal)
 		res, err := e.executeConstRow(ctx, index, c)
 		return res, errors.Wrap(err, "executeConstRow")
 	case "Limit":
+		statFn(CounterQueryLimitTotal)
 		res, err := e.executeLimitCall(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeLimitCall")
 	case "Percentile":
+		statFn(CounterQueryPercentileTotal)
 		res, err := e.executePercentile(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executePercentile")
 	case "Delete":
-		statFn() // TODO(twg) need this?
+		statFn(CounterQueryDeleteTotal)
 		res, err := e.executeDeleteRecords(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeDelete")
 	case "Sort":
+		statFn(CounterQuerySortTotal)
 		res, err := e.executeSort(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeSort")
 	case "Apply":
+		statFn(CounterQueryApplyTotal)
 		res, err := e.executeApply(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeApply")
 	case "Arrow":
+		statFn(CounterQueryArrowTotal)
 		res, err := e.executeArrow(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeArrow")
 	default: // e.g. "Row", "Union", "Intersect" or anything that returns a bitmap.
-		statFn()
 		res, err := e.executeBitmapCall(ctx, qcx, index, c, shards, opt)
 		return res, errors.Wrap(err, "executeBitmapCall")
 	}
@ -1506,13 +1515,42 @@ func (e *executor) executeBitmapCall(ctx context.Context, qcx *Qcx, index string
 	span.LogKV("pqlCallName", c.Name)
 	defer span.Finish()

-	indexTag := "index:" + index
-	metricName := "query_" + strings.ToLower(c.Name) + "_total"
-	if c.Name == "Row" && c.HasConditionArg() {
-		metricName = "query_row_bsi_total"
+	labels := prometheus.Labels{"index": index}
+	statFn := func(ctr *prometheus.CounterVec) {
+		if !opt.Remote {
+			ctr.With(labels).Inc()
+		}
 	}
+
 	if !opt.Remote {
-		e.Holder.Stats.CountWithCustomTags(metricName, 1, 1.0, []string{indexTag})
+		switch c.Name {
+		case "Row":
+			if c.HasConditionArg() {
+				statFn(CounterQueryRowBSITotal)
+			} else {
+				statFn(CounterQueryRowTotal)
+			}
+		case "Range":
+			statFn(CounterQueryRangeTotal)
+		case "Difference":
+			statFn(CounterQueryBitmapTotal)
+		case "Intersect":
+			statFn(CounterQueryIntersectTotal)
+		case "Union":
+			statFn(CounterQueryUnionTotal)
+		case "InnerUnionRows":
+			statFn(CounterQueryInnerUnionRowsTotal)
+		case "Xor":
+			statFn(CounterQueryXorTotal)
+		case "Not":
+			statFn(CounterQueryNotTotal)
+		case "Shift":
+			statFn(CounterQueryShiftTotal)
+		case "All":
+			statFn(CounterQueryAllTotal)
+		default:
+			statFn(CounterQueryBitmapTotal)
+		}
 	}

 	// Execute calls in bulk on each remote node and merge.
--- a/field.go
+++ b/field.go
@ -84,7 +84,6 @@ type Field struct {
 	viewMap map[string]*view

 	broadcaster broadcaster
-	Stats       stats.StatsClient
 	serializer  Serializer

 	// Field options.
@ -396,7 +395,6 @@ func newField(holder *Holder, path, index, name string, opts FieldOption) (*Fiel
 		viewMap: make(map[string]*view),

 		broadcaster: NopBroadcaster,
-		Stats:       stats.NopStatsClient,
 		serializer:  NopSerializer,

 		options: applyDefaultOptions(&fo),
@ -1165,7 +1163,6 @@ func (f *Field) newView(path, name string) *view {
 	view := newView(f.holder, path, f.index, f.name, name, f.options)
 	view.idx = f.idx
 	view.fld = f
-	view.stats = f.Stats
 	view.broadcaster = f.broadcaster
 	return view
 }
--- a/fragment.go
+++ b/fragment.go
@ -120,8 +120,6 @@ type fragment struct {
 	// mutexVector is used for mutex field types. It's checked for an
 	// existing value (to clear) prior to setting a new value.
 	mutexVector vector
-
-	stats stats.StatsClient
 }

 // newFragment returns a new instance of fragment.
@ -142,8 +140,6 @@ func newFragment(holder *Holder, idx *Index, fld *Field, vw *view, shard uint64)
 		CacheSize: DefaultCacheSize,

 		holder: holder,
-
-		stats: stats.NopStatsClient,
 	}
 	return f
 }
@ -405,7 +401,7 @@ func (f *fragment) unprotectedSetBit(tx Tx, rowID, columnID uint64) (changed boo
 		f.cache.Add(rowID, n)
 	}

-	f.stats.Count(MetricSetBit, 1, 1.0)
+	CounterSetBit.Inc()

 	return changed, nil
 }
@ -454,7 +450,7 @@ func (f *fragment) unprotectedClearBit(tx Tx, rowID, columnID uint64) (changed b
 		f.cache.Add(rowID, n)
 	}

-	f.stats.Count(MetricClearBit, 1, 1.0)
+	CounterClearBit.Inc()

 	return changed, nil
 }
@ -510,7 +506,7 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo
 		}
 	}

-	f.stats.Count("setRow", 1, 1.0)
+	CounterSetRow.Inc()

 	return changed, nil
 }
@ -1713,23 +1709,23 @@ func (p parallelSlices) Swap(i, j int) {
 // operations to the op log.
 func (f *fragment) importPositions(tx Tx, set, clear []uint64, rowSet map[uint64]struct{}) error {
 	if len(set) > 0 {
-		f.stats.Count(MetricImportingN, int64(len(set)), 1)
+		CounterImportingN.Add(float64(len(set)))

 		// TODO benchmark Add/RemoveN behavior with sorted/unsorted positions
 		changedN, err := tx.Add(f.index(), f.field(), f.view(), f.shard, set...)
 		if err != nil {
 			return errors.Wrap(err, "adding positions")
 		}
-		f.stats.Count(MetricImportedN, int64(changedN), 1)
+		CounterImportedN.Add(float64(changedN))
 	}

 	if len(clear) > 0 {
-		f.stats.Count(MetricClearingN, int64(len(clear)), 1)
+		CounterClearingingN.Add(float64(len(clear)))
 		changedN, err := tx.Remove(f.index(), f.field(), f.view(), f.shard, clear...)
 		if err != nil {
 			return errors.Wrap(err, "clearing positions")
 		}
-		f.stats.Count(MetricClearedN, int64(changedN), 1)
+		CounterClearedN.Add(float64(changedN))
 	}
 	return f.updateCaching(tx, rowSet)
 }
--- a/go.mod
+++ b/go.mod
@ -8,7 +8,7 @@ replace robpike.io/ivy => github.com/tgruben/ivy v0.0.0-20221107170120-634b546dc

 require (
 	github.com/CAFxX/gcnotifier v0.0.0-20220409005548-0153238b886a
-	github.com/DataDog/datadog-go v4.8.3+incompatible
+	github.com/DataDog/datadog-go v4.8.3+incompatible // indirect
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2
--- a/holder.go
+++ b/holder.go
@ -80,9 +80,6 @@ type Holder struct {
 	wg      sync.WaitGroup
 	closing chan struct{}

-	// Stats
-	Stats stats.StatsClient
-
 	// Data directory path.
 	path string

@ -257,7 +254,6 @@ type HolderConfig struct {
 	Schemator            disco.Schemator
 	Sharder              disco.Sharder
 	CacheFlushInterval   time.Duration
-	StatsClient          stats.StatsClient
 	Logger               logger.Logger

 	StorageConfig *storage.Config
@ -282,7 +278,6 @@ func DefaultHolderConfig() *HolderConfig {
 		Schemator:            disco.NewInMemSchemator(),
 		Sharder:              disco.InMemSharder,
 		CacheFlushInterval:   defaultCacheFlushInterval,
-		StatsClient:          stats.NopStatsClient,
 		Logger:               logger.NopLogger,
 		StorageConfig:        storage.NewDefaultConfig(),
 		RBFConfig:            rbfcfg.NewDefaultConfig(),
@ -324,7 +319,6 @@ func NewHolder(path string, cfg *HolderConfig) *Holder {
 		broadcaster: NopBroadcaster,

 		partitionN:           cfg.PartitionN,
-		Stats:                cfg.StatsClient,
 		cacheFlushInterval:   cfg.CacheFlushInterval,
 		OpenTranslateStore:   cfg.OpenTranslateStore,
 		OpenTranslateReader:  cfg.OpenTranslateReader,
@ -527,8 +521,6 @@ func (h *Holder) Open() error {
 	// Check if deletion was in progress when server was shutdown
 	h.processDeleteInflight()

-	h.Stats.Open()
-
 	h.opened.Close()

 	_ = testhook.Opened(h.Auditor, h, nil)
@ -629,8 +621,6 @@ func (h *Holder) Close() error {
 		fmt.Printf("%v\n", globalCallStats.report())
 	}

-	h.Stats.Close()
-
 	// Notify goroutines of closing and wait for completion.
 	close(h.closing)
 	h.wg.Wait()
@ -1157,7 +1147,6 @@ func (h *Holder) newIndex(path, name string) (*Index, error) {
 	if err != nil {
 		return nil, err
 	}
-	index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name()))
 	index.broadcaster = h.broadcaster
 	index.serializer = h.serializer
 	index.OpenTranslateStore = h.OpenTranslateStore
@ -1330,9 +1319,6 @@ type holderSyncer struct {

 	syncers errgroup.Group

-	// Stats
-	Stats stats.StatsClient
-
 	// Signals that the sync should stop.
 	Closing <-chan struct{}
 }
--- a/http_handler.go
+++ b/http_handler.go
@ -9,7 +9,6 @@ import (
 	"encoding/gob"
 	"encoding/hex"
 	"encoding/json"
-	"expvar"
 	"fmt"
 	"io"
 	"math"
@ -45,6 +44,7 @@ import (
 	"github.com/featurebasedb/featurebase/v3/storage"
 	"github.com/featurebasedb/featurebase/v3/tracing"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/prom2json"
@ -402,8 +402,7 @@ func (h *Handler) collectStats(next http.Handler) http.Handler {
 		next.ServeHTTP(w, r)
 		dur := time.Since(t)

-		statsTags := make([]string, 0, 5)
-
+		isSlow := "false"
 		longQueryTime := h.api.LongQueryTime()
 		if longQueryTime > 0 && dur > longQueryTime {
 			queryRequest := r.Context().Value(contextKeyQueryRequest)
@ -414,31 +413,27 @@ func (h *Handler) collectStats(next http.Handler) http.Handler {
 			}

 			h.logger.Printf("HTTP query duration %v exceeds %v: %s %s %s", dur, longQueryTime, r.Method, r.URL.String(), queryString)
-			statsTags = append(statsTags, "slow:true")
-		} else {
-			statsTags = append(statsTags, "slow:false")
+			isSlow = "true"
 		}

+		where := ""
 		pathParts := strings.Split(r.URL.Path, "/")
 		if externalPrefixFlag[pathParts[1]] {
-			statsTags = append(statsTags, "where:external")
+			where = "external"
 		} else {
-			statsTags = append(statsTags, "where:internal")
+			where = "internal"
 		}
-
-		statsTags = append(statsTags, "useragent:"+r.UserAgent())
-
 		path, err := mux.CurrentRoute(r).GetPathTemplate()
-		if err == nil {
-			statsTags = append(statsTags, "path:"+path)
-		}
-
-		statsTags = append(statsTags, "method:"+r.Method)
-
-		stats := h.api.StatsWithTags(statsTags)
-		if stats != nil {
-			stats.Timing(MetricHTTPRequest, dur, 0.1)
+		if err != nil {
+			path = ""
 		}
+		SummaryHttpRequests.With(prometheus.Labels{
+			"method":    r.Method,
+			"path":      path,
+			"slow":      isSlow,
+			"useragent": r.UserAgent(),
+			"where":     where,
+		}).Observe(dur.Seconds())
 	})
 }

@ -506,7 +501,6 @@ func newRouter(handler *Handler) http.Handler {
 	// TODO: figure out how to protect these if needed
 	router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
 	router.PathPrefix("/debug/fgprof").Handler(fgprof.Handler()).Methods("GET")
-	router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
 	router.Handle("/metrics", promhttp.Handler())

 	router.HandleFunc("/metrics.json", handler.chkAuthZ(handler.handleGetMetricsJSON, authz.Admin)).Methods("GET").Name("GetMetricsJSON")
@ -552,6 +546,8 @@ func newRouter(handler *Handler) http.Handler {
 	if handler.sqlEnabled {
 		router.HandleFunc("/sql", handler.chkAuthZ(handler.handlePostSQL, authz.Admin)).Methods("POST").Name("PostSQL")
 	}
+	// internal endpoint
+	router.HandleFunc("/sql", handler.chkAuthZ(handler.handlePostSQLPlanOperator, authz.Admin)).Headers("X-FeatureBase-Plan-Operator", "").Methods("POST").Name("PostSQLPlanOperator")

 	router.HandleFunc("/query-history", handler.chkAuthZ(handler.handleGetPastQueries, authz.Admin)).Methods("GET").Name("GetPastQueries")
 	router.HandleFunc("/version", handler.handleGetVersion).Methods("GET").Name("GetVersion")
@ -1393,10 +1389,70 @@ func (h *Handler) writeBadRequest(w http.ResponseWriter, r *http.Request, err er
 	}
 }

-// handlePostSQL handles /sql requests.
-func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
-	includePlan := false
+// handlePostSQLOperator handles an internal sql3 plan operator execution request
+// these requests come from other nodes in the cluster
+// handlePostSQLOperator will 'rehydrate' a plan operator and return data in the
+// featurebase wire format for effciency
+// we do not track these requests as user requests
+// TODO(pok) - thus is there anything we need here to align with how we do this for other nodes
+func (h *Handler) handlePostSQLPlanOperator(w http.ResponseWriter, r *http.Request) {

+	writeError := func(err error) {
+		if err != nil {
+			w.Write(wireprotocol.WriteError(err))
+		}
+	}
+
+	// always finish with a done message
+	defer w.Write(wireprotocol.WriteDone())
+
+	ctx := r.Context()
+
+	rootOperator, err := h.api.RehydratePlanOperator(ctx, r.Body)
+	if err != nil {
+		writeError(err)
+		return
+	}
+
+	// get a query iterator.
+	iter, err := rootOperator.Iterator(ctx, nil)
+	if err != nil {
+		writeError(err)
+		return
+	}
+	// read schema & write to response.
+	columns := rootOperator.Schema()
+	b, err := wireprotocol.WriteSchema(columns)
+	if err != nil {
+		writeError(err)
+		return
+	}
+	w.Write(b)
+
+	var rowErr error
+	var currentRow types.Row
+	var nextErr error
+
+	for currentRow, nextErr = iter.Next(ctx); nextErr == nil; currentRow, nextErr = iter.Next(ctx) {
+		b, err := wireprotocol.WriteRow(currentRow, columns)
+		if err != nil {
+			rowErr = err
+			break
+		}
+		w.Write(b)
+	}
+	if nextErr != nil && nextErr != types.ErrNoMoreRows {
+		rowErr = nextErr
+	}
+	writeError(rowErr)
+}
+
+// handlePostSQL handles /sql requests
+// supports a ?plan=true|false parameter to send back the plan in the
+// query response
+func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
+
+	includePlan := false
 	includePlanValue := r.URL.Query().Get("plan")
 	if len(includePlanValue) > 0 {
 		var err error
@ -1407,6 +1463,7 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
 		}
 	}

+	// get the body
 	b, err := io.ReadAll(r.Body)
 	if err != nil {
 		h.writeBadRequest(w, r, err)
@ -1420,6 +1477,9 @@ func (h *Handler) handlePostSQL(w http.ResponseWriter, r *http.Request) {
 	// put the requestId in the context
 	ctx := fbcontext.WithRequestID(r.Context(), requestID.String())

+	// update the counter for requests
+	PerfCounterSQLRequestSec.Add(1)
+
 	// Write response back to client.
 	w.Header().Set("Content-Type", "application/json")

--- a/idk/Makefile
+++ b/idk/Makefile
@ -231,7 +231,7 @@ TPKG ?= ./...
 test-run: testenv vendor
 	$(DOCKER_COMPOSE) build idk-test
 	$(DOCKER_COMPOSE) run -T idk-test bash -c "set -o pipefail; go test -v -mod=vendor -tags=odbc,dynamic $(TPKG) -covermode=atomic -coverpkg=$(TPKG) -coverprofile=/testdata/$(PROJECT)_base_coverage.out"
-	$(DOCKER_COMPOSE) run -T idk-test /go/src/github.com/molecula/featurebase/idk/reingest_test.sh
+	$(DOCKER_COMPOSE) run -T idk-test /go/src/github.com/featurebasedb/featurebase/idk/reingest_test.sh


 test-run-race: testenv vendor
--- a/idk/ingest.go
+++ b/idk/ingest.go
@ -55,7 +55,6 @@ const (
 )

 // TODO Jaeger
-// TODO Prometheus

 // Main holds all config for general ingest
 type Main struct {
@ -132,7 +131,6 @@ type Main struct {
 	newNexter func(c int) (IDAllocator, error)
 	ra        RangeAllocator

-	stats         stats.StatsClient
 	metricsServer *http.Server

 	log logger.Logger
@ -218,7 +216,7 @@ func NewMain() *Main {
 		Concurrency:      1,
 		CacheLength:      64,
 		PackBools:        "bools",
-		Namespace:        "ingester",
+		Namespace:        "ingester", // this is now ignored and hardcoded in metrics.go
 		IDAllocKeyPrefix: "ingest",

 		UseShardTransactionalEndpoint: os.Getenv("IDK_DEFAULT_SHARD_TRANSACTIONAL") != "",
@ -228,8 +226,6 @@ func NewMain() *Main {

 		SchemaManager: NopSchemaManager,

-		stats: stats.NopStatsClient,
-
 		log: logger.NewStandardLogger(os.Stderr),
 	}
 }
@ -457,7 +453,6 @@ initialFetch:
 				if v, ok := source.(Metadata); ok {
 					m.log.Printf("new schema - subject: %#v; version: %d; schema: %#v",
 						v.SchemaSubject(), v.SchemaVersion(), v.SchemaSchema())
-					// m.log.Printf("new schema: %#v", v.SchemaMetadata())
 				} else {
 					m.log.Printf("new schema: %#v", schema)
 				}
@ -465,7 +460,7 @@ initialFetch:
 				if err != nil {
 					return errors.Wrap(err, "batchFromSchema")
 				}
-				m.stats.Count(MetricIngesterSchemaChanges, 1, 1)
+				CounterIngesterSchemaChanges.Inc()
 				csvSlice = make([]string, len(schema))
 				if m.csvWriter != nil {
 					for i := range schema {
@ -571,7 +566,7 @@ initialFetch:
 		// skip bad rows only
 		if !rowHasError {
 			err = batch.Add(*row)
-			m.stats.Count(MetricIngesterRowsAdded, 1, 1)
+			CounterIngesterRowsAdded.Inc()
 		}

 		if err == pilosabatch.ErrBatchNowFull || err == pilosabatch.ErrBatchNowStale {
@ -958,7 +953,7 @@ func (m *Main) commitRecord(ctx context.Context, rec Record, limitCounter *msgCo
 		return errors.Wrap(err, "committing")
 	}
 	limitCounter.Increment(numRecords)
-	m.stats.Count(MetricCommittedRecords, int64(numRecords), 1)
+	CounterCommittedRecords.Add(float64(numRecords))
 	return nil
 }

@ -972,7 +967,7 @@ func (m *Main) NewLookupClient() (*PostgresClient, error) {
 func (m *Main) setupClient() (*tls.Config, error) {
 	var tlsConfig *tls.Config
 	var err error
-	var opts = []pilosaclient.ClientOption{pilosaclient.OptClientStatsClient(m.stats)}
+	var opts = []pilosaclient.ClientOption{}
 	if m.TLS.CertificatePath != "" {
 		tlsConfig, err = GetTLSConfig(&m.TLS, m.Log())
 		if err != nil {
@ -1037,34 +1032,24 @@ func (m *Main) setupClient() (*tls.Config, error) {
 }

 func (m *Main) setupStats() error {
-	if m.Stats != "" {
-		opts := []prometheus.ClientOption{prometheus.OptClientNamespace(m.Namespace)}
-		m.stats, _ = prometheus.NewPrometheusClient(opts...) // ignore error that must be nil
-
-		mux := http.NewServeMux()
-		// reg := prom.NewRegistry() // TODO switch to this once pilosa PrometheusClient is fixed and doesn't use the global registry internally.
-		// also change prom.DefaultGatherer to be "reg" at that time
-		reg := prom.DefaultRegisterer
-		promHandler := promhttp.InstrumentMetricHandler(reg, promhttp.HandlerFor(prom.DefaultGatherer, promhttp.HandlerOpts{}))
-		mux.Handle("/metrics", promHandler)
-
-		mux.Handle("/metrics.json", metricsJSONHandler{metricsURI: "http://" + m.Stats + "/metrics"})
-		m.metricsServer = &http.Server{Addr: m.Stats, Handler: mux}
-		ln, err := net.Listen("tcp", m.Stats)
-		if err != nil {
-			return errors.Wrapf(err, "listen for metrics on '%s'", m.Stats)
-		}
-
-		go func() {
-			m.log.Printf("Serving Prometheus metrics with namespace \"%s\" at %v/metrics\n", m.Namespace, m.Stats)
-			err = m.metricsServer.Serve(ln)
-			if err != http.ErrServerClosed {
-				m.log.Printf("serve metrics on '%s': %v", m.Stats, err)
-			}
-		}()
+	mux := http.NewServeMux()
+	promHandler := promhttp.InstrumentMetricHandler(prom.DefaultRegisterer, promhttp.HandlerFor(prom.DefaultGatherer, promhttp.HandlerOpts{}))
+	mux.Handle("/metrics", promHandler)
+	mux.Handle("/metrics.json", metricsJSONHandler{metricsURI: "http://" + m.Stats + "/metrics"})
+	m.metricsServer = &http.Server{Addr: m.Stats, Handler: mux}
+	ln, err := net.Listen("tcp", m.Stats)
+	if err != nil {
+		return errors.Wrapf(err, "listen for metrics on '%s'", m.Stats)
 	}
-	return nil

+	go func() {
+		m.log.Printf("Serving Prometheus metrics with namespace \"%s\" at %v/metrics\n", m.Namespace, m.Stats)
+		err = m.metricsServer.Serve(ln)
+		if err != http.ErrServerClosed {
+			m.log.Printf("serve metrics on '%s': %v", m.Stats, err)
+		}
+	}()
+	return nil
 }

 type metricsJSONHandler struct {
@ -1218,7 +1203,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error {
 				if err != nil {
 					return errors.Wrap(err, "clearing bools")
 				}
-				m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:packed-bool"})
+				CounterDeleterRowsAdded.With(prom.Labels{"type": "packed-bool"}).Inc()
 				continue
 			} else {
 				fieldName = directive
@ -1267,7 +1252,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error {
 				if err != nil {
 					return errors.Wrap(err, "clearing set")
 				}
-				m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:set"})
+				CounterDeleterRowsAdded.With(prom.Labels{"type": "set"}).Inc()
 			case pilosaclient.FieldTypeMutex:
 				if val == "" {
 					continue
@ -1278,7 +1263,7 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error {
 				if err != nil {
 					return errors.Wrap(err, "clearing mutex")
 				}
-				m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:mutex"})
+				CounterDeleterRowsAdded.With(prom.Labels{"type": "mutex"}).Inc()
 			case pilosaclient.FieldTypeBool:
 				_, err := client.Query(index.BatchQuery(
 					field.Clear(0, recordID),
@ -1287,19 +1272,19 @@ func (m *Main) runDeleter(c int, limitCounter *msgCounter) error {
 				if err != nil {
 					return errors.Wrap(err, "clearing bool")
 				}
-				m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:bool"})
+				CounterDeleterRowsAdded.With(prom.Labels{"type": "bool"}).Inc()
 			case pilosaclient.FieldTypeInt:
 				_, err := client.Query(field.Clear(0, recordID))
 				if err != nil {
 					return errors.Wrap(err, "clearing int")
 				}
-				m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:int"})
+				CounterDeleterRowsAdded.With(prom.Labels{"type": "int"}).Inc()
 			case pilosaclient.FieldTypeDecimal:
 				_, err := client.Query(field.Clear(0, recordID))
 				if err != nil {
 					return errors.Wrap(err, "clearing decimal")
 				}
-				m.stats.CountWithCustomTags(MetricDeleterRowsAdded, 1, 1, []string{"type:decimal"})
+				CounterDeleterRowsAdded.With(prom.Labels{"type": "decimal"}).Inc()
 			case pilosaclient.FieldTypeTime:
 				return errors.Errorf("deletion on time fields unimplemented")
 			default:
--- a/idk/ingest_test.go
+++ b/idk/ingest_test.go
@ -525,7 +525,7 @@ func TestIngesterServesPrometheusEndpoint(t *testing.T) {
 	if err != nil {
 		t.Errorf("read error: %v", err)
 	}
-	if strings.Contains(string(contents), MetricIngesterRowsAdded) {
+	if !strings.Contains(string(contents), MetricIngesterRowsAdded) {
 		t.Errorf("metric name missing: %v", MetricIngesterRowsAdded)
 	}
 	close(records)
--- a/idk/mds/importer.go
+++ b/idk/mds/importer.go
@ -248,8 +248,6 @@ func (m *importer) DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field
 	return fbClient.DoImport(string(qtbl.Key()), shard, path, data)
 }

-func (m *importer) StatsTiming(name string, value time.Duration, rate float64) {}
-
 // getQtbl takes a table (TableKey) and sets the local m.qtbl value. When we
 // originally set up this type, it was only used by IDK, and the table was known
 // at the beginning of the process, so it could be set on this import. But
--- a/idk/metrics.go
+++ b/idk/metrics.go
@ -1,8 +1,52 @@
 package idk

+import "github.com/prometheus/client_golang/prometheus"
+
 const (
 	MetricDeleterRowsAdded      = "deleter_rows_added_total"
 	MetricIngesterRowsAdded     = "ingester_rows_added_total"
 	MetricIngesterSchemaChanges = "ingester_schema_changes_total"
 	MetricCommittedRecords      = "committed_records"
 )
+
+var CounterIngesterSchemaChanges = prometheus.NewCounter(
+	prometheus.CounterOpts{
+		Namespace: "ingester",
+		Name:      MetricIngesterSchemaChanges,
+		Help:      "TODO",
+	},
+)
+
+var CounterIngesterRowsAdded = prometheus.NewCounter(
+	prometheus.CounterOpts{
+		Namespace: "ingester",
+		Name:      MetricIngesterRowsAdded,
+		Help:      "TODO",
+	},
+)
+
+var CounterCommittedRecords = prometheus.NewCounter(
+	prometheus.CounterOpts{
+		Namespace: "ingester",
+		Name:      MetricCommittedRecords,
+		Help:      "TODO",
+	},
+)
+
+var CounterDeleterRowsAdded = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Namespace: "ingester",
+		Name:      MetricDeleterRowsAdded,
+		Help:      "TODO",
+	},
+	[]string{
+		"type",
+	},
+)
+
+func init() {
+	prometheus.MustRegister(CounterIngesterSchemaChanges)
+	prometheus.MustRegister(CounterIngesterRowsAdded)
+	prometheus.MustRegister(CounterCommittedRecords)
+	prometheus.MustRegister(CounterDeleterRowsAdded)
+}
--- a/importer.go
+++ b/importer.go
@ -20,8 +20,6 @@ type Importer interface {
 	EncodeImportValues(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, vals []int64, ids []uint64, clear bool) (path string, data []byte, err error)
 	EncodeImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, vals, ids []uint64, clear bool) (path string, data []byte, err error)
 	DoImport(ctx context.Context, tid dax.TableID, fld *dax.Field, shard uint64, path string, data []byte) error
-
-	StatsTiming(name string, value time.Duration, rate float64)
 }

 // Ensure type implements interface.
--- a/index.go
+++ b/index.go
@ -20,6 +20,7 @@ import (
 	"github.com/featurebasedb/featurebase/v3/stats"
 	"github.com/featurebasedb/featurebase/v3/testhook"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/sync/errgroup"
 )

@ -44,7 +45,6 @@ type Index struct {

 	broadcaster broadcaster
 	serializer  Serializer
-	Stats       stats.StatsClient

 	// Passed to field for foreign-index lookup.
 	holder *Holder
@ -83,7 +83,6 @@ func NewIndex(holder *Holder, path, name string) (*Index, error) {
 		fields: make(map[string]*Field),

 		broadcaster:    NopBroadcaster,
-		Stats:          stats.NopStatsClient,
 		holder:         holder,
 		trackExistence: true,

@ -511,7 +510,7 @@ func (i *Index) AvailableShards(localOnly bool) *roaring.Bitmap {
 		b.UnionInPlace(f.AvailableShards(localOnly))
 	}

-	i.Stats.Gauge(MetricMaxShard, float64(b.Max()), 1.0)
+	GaugeIndexMaxShard.With(prometheus.Labels{"index": i.name}).Set(float64(b.Max()))
 	return b
 }

@ -924,7 +923,6 @@ func (i *Index) newField(path, name string) (*Field, error) {
 		return nil, err
 	}
 	f.idx = i
-	f.Stats = i.Stats
 	f.broadcaster = i.broadcaster
 	f.serializer = i.serializer
 	f.OpenTranslateStore = i.OpenTranslateStore
--- a/lattice/README.md
+++ b/lattice/README.md
@ -19,4 +19,4 @@ Open [http://localhost:3000](http://localhost:3000) to view it in the browser.

 Lattice can be embedded within the Pilosa binary, so the UI is fully accessible directly from the server, reducing operational complexity.

-If additional build dependencies `yarn` (`brew install yarn` and `brew upgrade yarn` perhaps) and `statik` (`make install-statik`) are available on your system, running `make generate-statik` before `make install` should produce a Pilosa binary with Lattice embedded. For up to date instructions, check the Pilosa [README](https://github.com/molecula/pilosa#getting-started).
+If additional build dependencies `yarn` (`brew install yarn` and `brew upgrade yarn` perhaps) and `statik` (`make install-statik`) are available on your system, running `make generate-statik` before `make install` should produce a Pilosa binary with Lattice embedded. For up to date instructions, check the Featurebase [README](https://github.com/featurebasedb/featurebase#getting-started).
--- a/metrics.go
+++ b/metrics.go
--- a/performancecounters.go
+++ b/performancecounters.go
@ -0,0 +1,203 @@
+package pilosa
+
+import (
+	"sync/atomic"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// PerformanceCounter holds data about a performance counter for external consumers
+type PerformanceCounter struct {
+	NameSpace   string
+	SubSystem   string
+	CounterName string
+	Help        string
+	Value       int64
+	CounterType int64
+}
+
+// constants for the counter types
+const (
+	// raw - for when you just want a count of something
+	CTR_TYPE_RAW = 0
+	// per second - for when you accumulate counts of things
+	// a consumer would sample this at intervals to arrive at a delta
+	// then divide by the time in seconds between the samples to get a
+	// per-second value
+	CTR_TYPE_PER_SECOND = 1
+	// ratio - for when you accumulate a count of something that you
+	// want to use as a numerator in a ratio calculation
+	// e.g. 'cache hits' could be a counter of this type and you could
+	// divide it by a 'cache lookups' counter to get the hit ratio (see below)
+	CTR_TYPE_RATIO = 2
+	// ratio base - for when you accumulate a count of something that you
+	// want to use as a denominator in a ratio calculation
+	// e.g. 'cache lookups' could be a counter of this type and you could
+	// use it as the denominator in a division with a 'cache hits' counter
+	// as the numerator to get the hit ratio
+	CTR_TYPE_RATIO_BASE = 3
+)
+
+type PerformanceCounters struct {
+	counterValues [5]perfCtrWrapper
+}
+
+type perfCtr struct {
+	nameSpace   string
+	subSystem   string
+	name        string
+	help        string
+	value       int64
+	counterType int64
+}
+
+func (p *perfCtr) Add(increment int64) {
+	atomic.AddInt64(&p.value, increment)
+}
+
+type perfCtrWrapper struct {
+	ctr *perfCtr
+	fun prometheus.CounterFunc
+}
+
+var PerfCounterSQLRequestSec = perfCtr{
+	nameSpace:   "pilosa",
+	subSystem:   "sql_statistics",
+	name:        "sql_requests_sec",
+	help:        "TODO",
+	value:       0,
+	counterType: CTR_TYPE_PER_SECOND,
+}
+
+var PerfCounterSQLInsertsSec = perfCtr{
+	nameSpace:   "pilosa",
+	subSystem:   "sql_statistics",
+	name:        "sql_inserts_sec",
+	help:        "TODO",
+	value:       0,
+	counterType: CTR_TYPE_PER_SECOND,
+}
+
+var PerfCounterSQLBulkInsertsSec = perfCtr{
+	nameSpace:   "pilosa",
+	subSystem:   "sql_statistics",
+	name:        "sql_bulk_inserts_sec",
+	help:        "TODO",
+	value:       0,
+	counterType: CTR_TYPE_PER_SECOND,
+}
+
+var PerfCounterSQLBulkInsertBatchesSec = perfCtr{
+	nameSpace:   "pilosa",
+	subSystem:   "sql_statistics",
+	name:        "sql_bulk_insert_batches_sec",
+	help:        "TODO",
+	value:       0,
+	counterType: CTR_TYPE_PER_SECOND,
+}
+
+var PerfCounterSQLDeletesSec = perfCtr{
+	nameSpace:   "pilosa",
+	subSystem:   "sql_statistics",
+	name:        "sql_deletes_sec",
+	help:        "TODO",
+	value:       0,
+	counterType: CTR_TYPE_PER_SECOND,
+}
+
+var PerfCounters *PerformanceCounters = newPerformanceCounters()
+
+func newPerformanceCounters() *PerformanceCounters {
+	ctrs := &PerformanceCounters{
+		counterValues: [5]perfCtrWrapper{
+			{
+				&PerfCounterSQLRequestSec,
+				prometheus.NewCounterFunc(
+					prometheus.CounterOpts{
+						Namespace: PerfCounterSQLRequestSec.nameSpace,
+						Subsystem: PerfCounterSQLRequestSec.subSystem,
+						Name:      PerfCounterSQLRequestSec.name,
+						Help:      PerfCounterSQLRequestSec.help,
+					},
+					func() float64 {
+						return float64(atomic.LoadInt64(&PerfCounterSQLRequestSec.value))
+					}),
+			},
+			{
+				&PerfCounterSQLInsertsSec,
+				prometheus.NewCounterFunc(
+					prometheus.CounterOpts{
+						Namespace: PerfCounterSQLInsertsSec.nameSpace,
+						Subsystem: PerfCounterSQLInsertsSec.subSystem,
+						Name:      PerfCounterSQLInsertsSec.name,
+						Help:      PerfCounterSQLInsertsSec.help,
+					},
+					func() float64 {
+						return float64(atomic.LoadInt64(&PerfCounterSQLInsertsSec.value))
+					}),
+			},
+			{
+				&PerfCounterSQLBulkInsertsSec,
+				prometheus.NewCounterFunc(
+					prometheus.CounterOpts{
+						Namespace: PerfCounterSQLBulkInsertsSec.nameSpace,
+						Subsystem: PerfCounterSQLBulkInsertsSec.subSystem,
+						Name:      PerfCounterSQLBulkInsertsSec.name,
+						Help:      PerfCounterSQLBulkInsertsSec.help,
+					},
+					func() float64 {
+						return float64(atomic.LoadInt64(&PerfCounterSQLBulkInsertsSec.value))
+					}),
+			},
+			{
+				&PerfCounterSQLBulkInsertBatchesSec,
+				prometheus.NewCounterFunc(
+					prometheus.CounterOpts{
+						Namespace: PerfCounterSQLBulkInsertBatchesSec.nameSpace,
+						Subsystem: PerfCounterSQLBulkInsertBatchesSec.subSystem,
+						Name:      PerfCounterSQLBulkInsertBatchesSec.name,
+						Help:      PerfCounterSQLBulkInsertBatchesSec.help,
+					},
+					func() float64 {
+						return float64(atomic.LoadInt64(&PerfCounterSQLBulkInsertBatchesSec.value))
+					}),
+			},
+			{
+				&PerfCounterSQLDeletesSec,
+				prometheus.NewCounterFunc(
+					prometheus.CounterOpts{
+						Namespace: PerfCounterSQLDeletesSec.nameSpace,
+						Subsystem: PerfCounterSQLDeletesSec.subSystem,
+						Name:      PerfCounterSQLDeletesSec.name,
+						Help:      PerfCounterSQLDeletesSec.help,
+					},
+					func() float64 {
+						return float64(atomic.LoadInt64(&PerfCounterSQLDeletesSec.value))
+					}),
+			},
+		},
+	}
+
+	for _, w := range ctrs.counterValues {
+		prometheus.MustRegister(w.fun)
+	}
+
+	return ctrs
+}
+
+// list all the counters
+// we can just read here without locking because if the counters get changed
+// midway thru the loop, absent evidence to the contrary, the world will not end
+func (p *PerformanceCounters) ListCounters() ([]PerformanceCounter, error) {
+	result := make([]PerformanceCounter, len(p.counterValues))
+	for i, c := range p.counterValues {
+		result[i] = PerformanceCounter{
+			NameSpace:   c.ctr.nameSpace,
+			SubSystem:   c.ctr.subSystem,
+			CounterName: c.ctr.name,
+			Value:       c.ctr.value,
+			CounterType: c.ctr.counterType,
+		}
+	}
+	return result, nil
+}
--- a/prometheus/prometheus.go
+++ b/prometheus/prometheus.go
@ -1,308 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package prometheus
-
-import (
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/featurebasedb/featurebase/v3/stats"
-	"github.com/prometheus/client_golang/prometheus"
-)
-
-const (
-	// namespace is prepended to each metric event name with "_"
-	defaultNamespace = "general"
-)
-
-// Ensure client implements interface.
-var _ stats.StatsClient = &prometheusClient{}
-
-// Module-level mutex to avoid copying in WithTags()
-var mu sync.Mutex
-
-// prometheusClient represents a Prometheus implementation of pilosa.statsClient.
-type prometheusClient struct {
-	tags        []string
-	logger      logger.Logger
-	counters    map[string]prometheus.Counter
-	counterVecs map[string]*prometheus.CounterVec
-	gauges      map[string]prometheus.Gauge
-	gaugeVecs   map[string]*prometheus.GaugeVec
-	observers   map[string]prometheus.Observer
-	summaryVecs map[string]*prometheus.SummaryVec
-	namespace   string
-}
-
-// ClientOption is a functional option type for prometheusClient
-type ClientOption func(c *prometheusClient)
-
-// OptClientPrefix is a functional option on prometheusClient used to set the namespace
-func OptClientNamespace(namespace string) ClientOption {
-	return func(c *prometheusClient) {
-		c.namespace = namespace
-	}
-}
-
-// NewPrometheusClient returns a new instance of StatsClient.
-func NewPrometheusClient(opts ...ClientOption) (*prometheusClient, error) {
-	client := &prometheusClient{
-		logger:      logger.NopLogger,
-		counters:    make(map[string]prometheus.Counter),
-		counterVecs: make(map[string]*prometheus.CounterVec),
-		gauges:      make(map[string]prometheus.Gauge),
-		gaugeVecs:   make(map[string]*prometheus.GaugeVec),
-		observers:   make(map[string]prometheus.Observer),
-		summaryVecs: make(map[string]*prometheus.SummaryVec),
-		namespace:   defaultNamespace,
-	}
-
-	for _, opt := range opts {
-		opt(client)
-	}
-
-	return client, nil
-}
-
-// Open no-op to satisfy interface
-func (c *prometheusClient) Open() {}
-
-// Close no-op to satisfy interface
-func (c *prometheusClient) Close() error {
-	return nil
-}
-
-// Tags returns a sorted list of tags on the client.
-func (c *prometheusClient) Tags() []string {
-	return c.tags
-}
-
-// labels returns an instance of prometheus.Labels with the value of the set tags.
-func (c *prometheusClient) labels() prometheus.Labels {
-	return tagsToLabels(c.tags, c.logger)
-}
-
-// WithTags returns a new client with additional tags appended.
-func (c *prometheusClient) WithTags(tags ...string) stats.StatsClient {
-	return &prometheusClient{
-		tags:        unionStringSlice(c.tags, tags),
-		logger:      c.logger,
-		counters:    c.counters,
-		counterVecs: c.counterVecs,
-		gauges:      c.gauges,
-		gaugeVecs:   c.gaugeVecs,
-		observers:   c.observers,
-		summaryVecs: c.summaryVecs,
-		namespace:   c.namespace,
-	}
-}
-
-// Count tracks the number of times something occurs per second.
-func (c *prometheusClient) Count(name string, value int64, rate float64) {
-	mu.Lock()
-	defer mu.Unlock()
-
-	var counter prometheus.Counter
-	var ok bool
-	name = strings.Replace(name, ".", "_", -1)
-	labels := c.labels()
-	opts := prometheus.CounterOpts{
-		Namespace: c.namespace,
-		Name:      name,
-	}
-	if len(labels) == 0 {
-		counter, ok = c.counters[name]
-		if !ok {
-			counter = prometheus.NewCounter(opts)
-			c.counters[name] = counter
-			prometheus.MustRegister(counter)
-		}
-	} else {
-		var counterVec *prometheus.CounterVec
-		counterVec, ok = c.counterVecs[name]
-		if !ok {
-			counterVec = prometheus.NewCounterVec(
-				opts,
-				labelKeys(labels),
-			)
-			c.counterVecs[name] = counterVec
-			prometheus.MustRegister(counterVec)
-		}
-		var err error
-		counter, err = counterVec.GetMetricWith(labels)
-		if err != nil {
-			c.logger.Errorf("counterVec.GetMetricWith error: %s", err)
-		}
-	}
-	if value == 1 {
-		counter.Inc()
-	} else {
-		counter.Add(float64(value))
-	}
-}
-
-// CountWithCustomTags tracks the number of times something occurs per second with custom tags.
-func (c *prometheusClient) CountWithCustomTags(name string, value int64, rate float64, t []string) {
-	c.WithTags(append(c.tags, t...)...).Count(name, value, rate)
-}
-
-// Gauge sets the value of a metric.
-func (c *prometheusClient) Gauge(name string, value float64, rate float64) {
-	mu.Lock()
-	defer mu.Unlock()
-
-	var gauge prometheus.Gauge
-	var ok bool
-	name = strings.Replace(name, ".", "_", -1)
-	labels := c.labels()
-	opts := prometheus.GaugeOpts{
-		Namespace: c.namespace,
-		Name:      name,
-	}
-	if len(labels) == 0 {
-		gauge, ok = c.gauges[name]
-		if !ok {
-			gauge = prometheus.NewGauge(opts)
-			c.gauges[name] = gauge
-			prometheus.MustRegister(gauge)
-		}
-	} else {
-		var gaugeVec *prometheus.GaugeVec
-		gaugeVec, ok = c.gaugeVecs[name]
-		if !ok {
-			gaugeVec = prometheus.NewGaugeVec(
-				opts,
-				labelKeys(labels),
-			)
-			c.gaugeVecs[name] = gaugeVec
-			prometheus.MustRegister(gaugeVec)
-		}
-		var err error
-		gauge, err = gaugeVec.GetMetricWith(labels)
-		if err != nil {
-			c.logger.Errorf("gaugeVec.GetMetricWith error: %s", err)
-			return
-		}
-	}
-	gauge.Set(float64(value))
-}
-
-// Histogram tracks statistical distribution of a metric.
-func (c *prometheusClient) Histogram(name string, value float64, rate float64) {
-	mu.Lock()
-	defer mu.Unlock()
-
-	var observer prometheus.Observer
-	var ok bool
-	name = strings.Replace(name, ".", "_", -1)
-	labels := c.labels()
-	opts := prometheus.SummaryOpts{
-		Namespace:  c.namespace,
-		Name:       name,
-		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
-	}
-	if len(labels) == 0 {
-		observer, ok = c.observers[name]
-		if !ok {
-			summary := prometheus.NewSummary(opts)
-			observer = summary
-			c.observers[name] = observer
-			prometheus.MustRegister(summary)
-		}
-	} else {
-		var summaryVec *prometheus.SummaryVec
-		summaryVec, ok = c.summaryVecs[name]
-		if !ok {
-			summaryVec = prometheus.NewSummaryVec(
-				opts,
-				labelKeys(labels),
-			)
-			c.summaryVecs[name] = summaryVec
-			prometheus.MustRegister(summaryVec)
-		}
-		var err error
-		observer, err = summaryVec.GetMetricWith(labels)
-		if err != nil {
-			c.logger.Errorf("summaryVec.GetMetricWith error: %s", err)
-			return
-		}
-	}
-	observer.Observe(value)
-}
-
-// Set tracks number of unique elements.
-func (c *prometheusClient) Set(name string, value string, rate float64) {
-	c.logger.Infof("prometheusClient.Set unimplemented: %s=%s", name, value)
-}
-
-// Timing tracks timing information for a metric.
-func (c *prometheusClient) Timing(name string, value time.Duration, rate float64) {
-	c.Histogram(name, value.Seconds(), rate)
-}
-
-// SetLogger sets the logger for client.
-func (c *prometheusClient) SetLogger(logger logger.Logger) {
-	c.logger = logger
-}
-
-// unionStringSlice returns a sorted set of tags which combine a & b.
-func unionStringSlice(a, b []string) []string {
-	// Sort both sets first.
-	sort.Strings(a)
-	sort.Strings(b)
-
-	// Find size of largest slice.
-	n := len(a)
-	if len(b) > n {
-		n = len(b)
-	}
-
-	// Exit if both sets are empty.
-	if n == 0 {
-		return nil
-	}
-
-	// Iterate over both in order and merge.
-	other := make([]string, 0, n)
-	for len(a) > 0 || len(b) > 0 {
-		if len(a) == 0 {
-			other, b = append(other, b[0]), b[1:]
-		} else if len(b) == 0 {
-			other, a = append(other, a[0]), a[1:]
-		} else if a[0] < b[0] {
-			other, a = append(other, a[0]), a[1:]
-		} else if b[0] < a[0] {
-			other, b = append(other, b[0]), b[1:]
-		} else {
-			other, a, b = append(other, a[0]), a[1:], b[1:]
-		}
-	}
-	return other
-}
-
-func tagsToLabels(tags []string, logger logger.Logger) (labels prometheus.Labels) {
-	labels = make(prometheus.Labels)
-	for _, tag := range tags {
-		tagParts := strings.SplitAfterN(tag, ":", 2)
-		if len(tagParts) != 2 {
-			// only process tags in "key:value" form
-			logger.Errorf("invalid Prometheus label: %v\n", tag)
-			continue
-		}
-		labels[tagParts[0][0:len(tagParts[0])-1]] = tagParts[1]
-	}
-	return labels
-}
-
-func labelKeys(labels prometheus.Labels) (keys []string) {
-	keys = make([]string, len(labels))
-	i := 0
-	for k := range labels {
-		keys[i] = k
-		i++
-	}
-	return keys
-}
--- a/prometheus/prometheus_test.go
+++ b/prometheus/prometheus_test.go
@ -3,62 +3,32 @@
 package prometheus_test

 import (
-	"reflect"
 	"testing"
-	"time"

 	pilosaPrometheus "github.com/featurebasedb/featurebase/v3/prometheus"
 	"github.com/prometheus/client_golang/prometheus"
 	io_prometheus_client "github.com/prometheus/client_model/go"
 )

-func TestPrometheusClient_WithTags(t *testing.T) {
-	// Create a new client.
-	c, err := pilosaPrometheus.NewPrometheusClient()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer c.Close()
-
-	// Create a new client with additional tags.
-	c1 := c.WithTags("foo", "bar")
-	if tags := c1.Tags(); !reflect.DeepEqual(tags, []string{"bar", "foo"}) {
-		t.Fatalf("unexpected tags: %+v", tags)
-	}
-
-	// Create a new client from the clone with more tags.
-	c2 := c1.WithTags("bar", "baz")
-	if tags := c2.Tags(); !reflect.DeepEqual(tags, []string{"bar", "baz", "foo"}) {
-		t.Fatalf("unexpected tags: %+v", tags)
-	}
-}
-
 func TestPrometheusClient_Methods(t *testing.T) {
-	// Create a new client.
-	c, err := pilosaPrometheus.NewPrometheusClient(
-		pilosaPrometheus.OptClientNamespace("testns"),
-	)
-	if err != nil {
-		t.Fatal(err)
-	}
+	c := test.MustRunCluster(t, 1)
 	defer c.Close()

-	dur, _ := time.ParseDuration("123us")
-	c.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"})
-	c.Count("cc", 1, 1.0)
-	c.Gauge("gg", 10, 1.0)
-	c.Histogram("hh", 1, 1.0)
-	c.Timing("tt", dur, 1.0)
-
 	metricFams, err := prometheus.DefaultGatherer.Gather()
 	if err != nil {
 		t.Fatal(err)
 	}
-	for _, metricName := range []string{"testns_ct", "testns_cc", "testns_gg", "testns_hh", "testns_tt"} {
+	for _, metricName := range []string{
+		"pilosa_sql_statistics_sql_bulk_insert_batches_sec",
+		"pilosa_sql_statistics_sql_bulk_inserts_sec",
+		"pilosa_sql_statistics_sql_deletes_sec",
+		"pilosa_sql_statistics_sql_inserts_sec",
+		"pilosa_sql_statistics_sql_requests_sec",
+	} {
 		if metricExists(metricName, metricFams) {
 			continue
 		}
-		t.Fatalf("Metric was not recorded: %s", metricName)
+		t.Fatalf("metric does not exist: %s", metricName)
 	}
 }

--- a/row.go
+++ b/row.go
@ -362,7 +362,7 @@ func (r *Row) Difference(others ...*Row) *Row {
 // be incorrect.
 //
 // Why unsupported? For a full description, see:
-// https://github.com/molecula/pilosa/issues/403.
+// https://github.com/featurebasedb/pilosa/issues/403.
 // In short, the current implementation will shift a bit
 // at the edge of a shard out of the shard and into a
 // container which is assumed to be an invalid container
@ -612,7 +612,7 @@ func (s *RowSegment) Xor(other *RowSegment) *RowSegment {
 // Shift returns s shifted by 1 bit.
 func (s *RowSegment) Shift() (*RowSegment, error) {
 	// TODO: deal with overflow
-	// See issue: https://github.com/molecula/pilosa/issues/403
+	// See issue: https://github.com/featurebasedb/pilosa/issues/403
 	data, err := s.data.Shift(1)
 	if err != nil {
 		return nil, errors.Wrap(err, "shifting roaring data")
--- a/server.go
+++ b/server.go
@ -5,6 +5,7 @@ package pilosa
 import (
 	"context"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"os/exec"
@ -246,15 +247,6 @@ func OptServerPrimaryTranslateStore(store TranslateStore) ServerOption {
 	}
 }

-// OptServerStatsClient is a functional option on Server
-// used to specify the stats client.
-func OptServerStatsClient(sc stats.StatsClient) ServerOption {
-	return func(s *Server) error {
-		s.holderConfig.StatsClient = sc
-		return nil
-	}
-}
-
 // OptServerDiagnosticsInterval is a functional option on Server
 // used to specify the duration between diagnostic checks.
 func OptServerDiagnosticsInterval(dur time.Duration) ServerOption {
@ -542,7 +534,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 		return nil, err
 	}
 	s.holder = NewHolder(path, s.holderConfig)
-	s.holder.Stats.SetLogger(s.logger)
 	cwd, err := os.Getwd()
 	if err != nil {
 		return nil, err
@ -558,9 +549,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
 	s.cluster.sharder = s.sharder
 	s.cluster.serverlessStorage = s.serverlessStorage

-	// Append the NodeID tag to stats.
-	s.holder.Stats = s.holder.Stats.WithTags(fmt.Sprintf("node_id:%s", s.nodeID))
-
 	s.executor.Holder = s.holder
 	s.holder.executor = s.executor
 	s.executor.Cluster = s.cluster
@ -658,7 +646,6 @@ func (s *Server) Open() error {
 	s.syncer.Node = node
 	s.syncer.Cluster = s.cluster
 	s.syncer.Closing = s.closing
-	s.syncer.Stats = s.holder.Stats.WithTags("component:HolderSyncer")

 	// Start background process listening for translation
 	// sync resets.
@ -1250,26 +1237,26 @@ func (s *Server) monitorRuntime() {
 			return
 		case <-s.gcNotifier.AfterGC():
 			// GC just ran.
-			s.holder.Stats.Count(MetricGarbageCollection, 1, 1.0)
+			CounterGarbageCollection.Inc()
 		case <-ticker.C:
 		}

 		// Record the number of go routines.
-		s.holder.Stats.Gauge(MetricGoroutines, float64(runtime.NumGoroutine()), 1.0)
+		GaugeGoroutines.Set(float64(runtime.NumGoroutine()))

 		openFiles, err := countOpenFiles()
 		// Open File handles.
 		if err == nil {
-			s.holder.Stats.Gauge(MetricOpenFiles, float64(openFiles), 1.0)
+			GaugeOpenFiles.Set(float64(openFiles))
 		}

 		// Runtime memory metrics.
 		runtime.ReadMemStats(&m)
-		s.holder.Stats.Gauge(MetricHeapAlloc, float64(m.HeapAlloc), 1.0)
-		s.holder.Stats.Gauge(MetricHeapInuse, float64(m.HeapInuse), 1.0)
-		s.holder.Stats.Gauge(MetricStackInuse, float64(m.StackInuse), 1.0)
-		s.holder.Stats.Gauge(MetricMallocs, float64(m.Mallocs), 1.0)
-		s.holder.Stats.Gauge(MetricFrees, float64(m.Frees), 1.0)
+		GaugeHeapAlloc.Set(float64(m.HeapAlloc))
+		GaugeHeapInUse.Set(float64(m.HeapInuse))
+		GaugeStackInUse.Set(float64(m.StackInuse))
+		GaugeMallocs.Set(float64(m.Mallocs))
+		GaugeFrees.Set(float64(m.Frees))
 	}
 }

@ -1413,6 +1400,10 @@ func (s *Server) CompileExecutionPlan(ctx context.Context, q string) (planner_ty
 	return s.executionPlannerFn(s.executor, s.executor.client.api, q).CompilePlan(ctx, st)
 }

+func (s *Server) RehydratePlanOperator(ctx context.Context, reader io.Reader) (planner_types.PlanOperator, error) {
+	return s.executionPlannerFn(s.executor, s.executor.client.api, "").RehydratePlanOp(ctx, reader)
+}
+
 // countOpenFiles on operating systems that support lsof.
 func countOpenFiles() (int, error) {
 	switch runtime.GOOS {
--- a/server/grpc.go
+++ b/server/grpc.go
@ -41,12 +41,11 @@ type GRPCHandler struct {
 	perms             *authz.GroupPermissions
 	logger            logger.Logger
 	queryLogger       logger.Logger
-	stats             stats.StatsClient
 	inspectDeprecated sync.Once
 }

 func NewGRPCHandler(api *pilosa.API) *GRPCHandler {
-	return &GRPCHandler{api: api, logger: logger.NopLogger, stats: stats.NopStatsClient}
+	return &GRPCHandler{api: api, logger: logger.NopLogger}
 }

 func (h *GRPCHandler) WithLogger(logger logger.Logger) *GRPCHandler {
@ -54,11 +53,6 @@ func (h *GRPCHandler) WithLogger(logger logger.Logger) *GRPCHandler {
 	return h
 }

-func (h *GRPCHandler) WithStats(stats stats.StatsClient) *GRPCHandler {
-	h.stats = stats
-	return h
-}
-
 func (h *GRPCHandler) WithPerms(perms *authz.GroupPermissions) *GRPCHandler {
 	h.perms = perms
 	return h
@ -140,7 +134,7 @@ func errToStatusError(err error) error {
 }

 func (h *GRPCHandler) execSQL(ctx context.Context, queryStr string) (pb.ToRowser, error) {
-	h.stats.Count(pilosa.MetricSqlQueries, 1, 1)
+	pilosa.CounterSQLQueries.Inc()
 	return execSQL(ctx, h.api, h.logger, queryStr)
 }

@ -340,9 +334,9 @@ func (h *GRPCHandler) QueryPQL(req *pb.QueryPQLRequest, stream pb.Pilosa_QueryPQ
 		return errToStatusError(err)
 	}
 	durFormat := time.Since(t)
-	h.stats.Timing(pilosa.MetricGRPCStreamQueryDurationSeconds, durQuery, 0.1)
-	h.stats.Timing(pilosa.MetricGRPCStreamFormatDurationSeconds, durFormat, 0.1)
-	h.stats.Count(pilosa.MetricPqlQueries, 1, 1)
+	pilosa.SummaryGRPCStreamQueryDurationSeconds.Observe(durQuery.Seconds())
+	pilosa.SummaryGRPCStreamFormatDurationSeconds.Observe(durFormat.Seconds())
+	pilosa.CounterPQLQueries.Inc()

 	return errToStatusError(nil)
 }
@ -406,9 +400,9 @@ func (h *GRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest
 		return nil, errors.Wrap(err, "sending header")
 	}

-	h.stats.Timing(pilosa.MetricGRPCUnaryQueryDurationSeconds, durQuery, 0.1)
-	h.stats.Timing(pilosa.MetricGRPCUnaryFormatDurationSeconds, durFormat, 0.1)
-	h.stats.Count(pilosa.MetricPqlQueries, 1, 1)
+	pilosa.SummaryGRPCUnaryQueryDurationSeconds.Observe(durQuery.Seconds())
+	pilosa.SummaryGRPCUnaryFormatDurationSeconds.Observe(durFormat.Seconds())
+	pilosa.CounterPQLQueries.Inc()

 	return table, errToStatusError(nil)
 }
@ -510,11 +504,10 @@ type VDSMGRPCHandler struct {
 	grpcHandler *GRPCHandler
 	api         *pilosa.API
 	logger      logger.Logger
-	stats       stats.StatsClient
 }

 func NewVDSMGRPCHandler(grpcHandler *GRPCHandler, api *pilosa.API) *VDSMGRPCHandler {
-	return &VDSMGRPCHandler{grpcHandler: grpcHandler, api: api, logger: logger.NopLogger, stats: stats.NopStatsClient}
+	return &VDSMGRPCHandler{grpcHandler: grpcHandler, api: api, logger: logger.NopLogger}
 }

 func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler {
@ -522,11 +515,6 @@ func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler {
 	return h
 }

-func (h *VDSMGRPCHandler) WithStats(stats stats.StatsClient) *VDSMGRPCHandler {
-	h.stats = stats
-	return h
-}
-
 // GetVDSs returns a single VDS given a name
 func (h *VDSMGRPCHandler) GetVDS(ctx context.Context, req *vdsm_pb.GetVDSRequest) (*vdsm_pb.GetVDSResponse, error) {
 	typedIdOrName := req.GetIdOrName()
@ -1481,7 +1469,6 @@ type grpcServer struct {

 	logger      logger.Logger
 	queryLogger logger.Logger
-	stats       stats.StatsClient
 }

 type grpcServerOption func(s *grpcServer) error
@ -1514,13 +1501,6 @@ func OptGRPCServerLogger(logger logger.Logger) grpcServerOption {
 	}
 }

-func OptGRPCServerStats(stats stats.StatsClient) grpcServerOption {
-	return func(s *grpcServer) error {
-		s.stats = stats
-		return nil
-	}
-}
-
 func OptGRPCServerAuth(authn *authn.Auth) grpcServerOption {
 	return func(s *grpcServer) error {
 		s.auth = authn
@ -1640,7 +1620,7 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) {

 	// create grpc server
 	server.grpcServer = grpc.NewServer(gopts...)
-	grpcHandler := NewGRPCHandler(server.api).WithLogger(server.logger).WithStats(server.stats).WithQueryLogger(server.queryLogger)
+	grpcHandler := NewGRPCHandler(server.api).WithLogger(server.logger).WithQueryLogger(server.queryLogger)

 	// add server permissions if we've got 'em
 	if server.perms != nil {
@ -1648,7 +1628,7 @@ func NewGRPCServer(opts ...grpcServerOption) (*grpcServer, error) {
 	}

 	pb.RegisterPilosaServer(server.grpcServer, grpcHandler)
-	vdsm_pb.RegisterMoleculaServer(server.grpcServer, NewVDSMGRPCHandler(grpcHandler, server.api).WithLogger(server.logger).WithStats(server.stats))
+	vdsm_pb.RegisterMoleculaServer(server.grpcServer, NewVDSMGRPCHandler(grpcHandler, server.api).WithLogger(server.logger))

 	// register the server so its services are available to grpc_cli and others
 	reflection.Register(server.grpcServer)
--- a/server/handler_test.go
+++ b/server/handler_test.go
@ -1080,15 +1080,6 @@ func TestHandler_Endpoints(t *testing.T) {
 		}
 	})

-	t.Run("Expvars", func(t *testing.T) {
-		w := httptest.NewRecorder()
-		r := test.MustNewHTTPRequest("GET", "/debug/vars", nil)
-		h.ServeHTTP(w, r)
-		if w.Code != http.StatusOK {
-			t.Fatalf("unexpected status code: %d", w.Code)
-		}
-	})
-
 	t.Run("Recalculate Caches", func(t *testing.T) {
 		w := httptest.NewRecorder()
 		h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil))
--- a/server/server.go
+++ b/server/server.go
@ -474,11 +474,6 @@ func (m *Command) setupServer() error {
 		diagnosticsInterval = defaultDiagnosticsInterval
 	}

-	statsClient, err := newStatsClient(m.Config.Metric.Service, m.Config.Metric.Host, m.Config.Namespace())
-	if err != nil {
-		return errors.Wrap(err, "new stats client")
-	}
-
 	if m.Config.Listener == nil {
 		m.ln, err = getListener(*uri, m.tlsConfig)
 		if err != nil {
@ -581,7 +576,6 @@ func (m *Command) setupServer() error {
 		pilosa.OptServerQueryLogger(m.queryLogger),
 		pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),
 		pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
-		pilosa.OptServerStatsClient(statsClient),
 		pilosa.OptServerURI(advertiseURI),
 		pilosa.OptServerGRPCURI(advertiseGRPCURI),
 		pilosa.OptServerClusterName(m.Config.Cluster.Name),
@ -691,7 +685,6 @@ func (m *Command) setupServer() error {
 		OptGRPCServerListener(m.grpcLn),
 		OptGRPCServerTLSConfig(m.tlsConfig),
 		OptGRPCServerLogger(m.logger),
-		OptGRPCServerStats(statsClient),
 		OptGRPCServerAuth(m.auth),
 		OptGRPCServerPerm(&p),
 		OptGRPCServerQueryLogger(m.queryLogger),
@ -828,24 +821,6 @@ func (m *Command) Close() error {
 	}
 }

-// newStatsClient creates a stats client from the config
-func newStatsClient(name string, host string, namespace string) (stats.StatsClient, error) {
-	switch name {
-	case "expvar":
-		return stats.NewExpvarStatsClient(), nil
-	case "statsd":
-		return statsd.NewStatsClient(host, namespace)
-	case "prometheus":
-		return prometheus.NewPrometheusClient(
-			prometheus.OptClientNamespace(namespace),
-		)
-	case "nop", "none":
-		return stats.NopStatsClient, nil
-	default:
-		return nil, errors.Errorf("'%v' not a valid stats client, choose from [expvar, statsd, prometheus, none].", name)
-	}
-}
-
 // getListener gets a net.Listener based on the config.
 func getListener(uri pnet.URI, tlsconf *tls.Config) (ln net.Listener, err error) {
 	// If bind URI has the https scheme, enable TLS
--- a/sql3/interfaces.go
+++ b/sql3/interfaces.go
@ -3,6 +3,7 @@ package sql3

 import (
 	"context"
+	"io"

 	"github.com/featurebasedb/featurebase/v3/sql3/parser"
 	"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
@ -10,6 +11,7 @@ import (

 type CompilePlanner interface {
 	CompilePlan(context.Context, parser.Statement) (types.PlanOperator, error)
+	RehydratePlanOp(context.Context, io.Reader) (types.PlanOperator, error)
 }

 // Ensure type implements interface.
@ -25,3 +27,7 @@ func NewNopCompilePlanner() *NopCompilePlanner {
 func (p *NopCompilePlanner) CompilePlan(ctx context.Context, stmt parser.Statement) (types.PlanOperator, error) {
 	return nil, nil
 }
+
+func (p *NopCompilePlanner) RehydratePlanOp(ctx context.Context, reader io.Reader) (types.PlanOperator, error) {
+	return nil, nil
+}
--- a/sql3/planner/compilecreatetable.go
+++ b/sql3/planner/compilecreatetable.go
@ -196,7 +196,13 @@ func (p *ExecutionPlanner) compileColumn(col *parser.ColumnDefinition) (*createT
 	case dax.BaseTypeBool:
 		column.fos = append(column.fos, pilosa.OptFieldTypeBool())
 	case dax.BaseTypeDecimal:
-		// Get the scale value.
+
+		// if we don't have a scale, it's an error
+		if col.Type.Scale == nil {
+			return nil, sql3.NewErrDecimalScaleExpected(col.Type.Name.NamePos.Line, col.Type.Name.NamePos.Column)
+		}
+
+		// get the scale value
 		scale, err = strconv.ParseInt(col.Type.Scale.Value, 10, 64)
 		if err != nil {
 			return nil, err
--- a/sql3/planner/compileselect.go
+++ b/sql3/planner/compileselect.go
@ -302,11 +302,16 @@ func (p *ExecutionPlanner) compileSource(scope *PlanOpQuery, source parser.Sourc
 		// doing this check here because we don't have a 'system' flag that exists in the FB schema
 		st, ok := systemTables[strings.ToLower(tableName)]
 		if ok {
+			var op types.PlanOperator
+			op = NewPlanOpSystemTable(p, st)
+			if st.requiresFanout {
+				op = NewPlanOpFanout(p, op)
+			}
 			if sourceExpr.Alias != nil {
 				aliasName := parser.IdentName(sourceExpr.Alias)
-				return NewPlanOpRelAlias(aliasName, NewPlanOpSystemTable(p, st)), nil
+				return NewPlanOpRelAlias(aliasName, op), nil
 			}
-			return NewPlanOpSystemTable(p, st), nil
+			return op, nil

 		}
 		// get all the columns for this table - we will eliminate unused ones
--- a/sql3/planner/executionplanner.go
+++ b/sql3/planner/executionplanner.go
@ -3,7 +3,12 @@
 package planner

 import (
+	"bytes"
 	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"

 	pilosa "github.com/featurebasedb/featurebase/v3"
 	"github.com/featurebasedb/featurebase/v3/logger"
@ -79,6 +84,20 @@ func (p *ExecutionPlanner) CompilePlan(ctx context.Context, stmt parser.Statemen
 	return rootOperator, err
 }

+func (p *ExecutionPlanner) RehydratePlanOp(ctx context.Context, reader io.Reader) (types.PlanOperator, error) {
+	rdr := newWireProtocolParser(p, reader)
+	message, err := rdr.nextMessage()
+	if err != nil {
+		return nil, err
+	}
+	switch m := message.(type) {
+	case *messagePlanOp:
+		return m.op, nil
+	default:
+		return nil, sql3.NewErrInternalf("unexpected message type '%T'", message)
+	}
+}
+
 func (p *ExecutionPlanner) analyzePlan(stmt parser.Statement) error {
 	switch stmt := stmt.(type) {
 	case *parser.SelectStatement:
@ -120,3 +139,216 @@ const (
 func (p *ExecutionPlanner) checkAccess(ctx context.Context, objectName string, _ accessType) error {
 	return nil
 }
+
+type reduceFunc func(ctx context.Context, prev, v types.Rows) (types.Rows, error)
+
+type mapResponse struct {
+	node   pilosa.ClusterNode
+	result types.Rows
+	err    error
+}
+
+func (e *ExecutionPlanner) mapReducePlanOp(ctx context.Context, op types.PlanOperator, reduceFn reduceFunc) (result types.Rows, err error) {
+	ch := make(chan mapResponse)
+
+	// Wrap context with a cancel to kill goroutines on exit.
+	ctx, cancel := context.WithCancel(ctx)
+	// Create an errgroup so we can wait for all the goroutines to exit
+	eg, ctx := errgroup.WithContext(ctx)
+
+	// After we're done processing, we have to wait for any outstanding
+	// functions in the ErrGroup to complete. If we didn't have an error
+	// already at that point, we'll report any errors from the ErrGroup
+	// instead.
+	defer func() {
+		cancel()
+		errWait := eg.Wait()
+		if err == nil {
+			err = errWait
+		}
+	}()
+
+	nodes := e.systemAPI.ClusterNodes()
+
+	// Start mapping across all nodes
+	if err = e.mapper(ctx, eg, ch, nodes, op, reduceFn); err != nil {
+		return nil, errors.Wrap(err, "starting mapper")
+	}
+
+	// Iterate over all map responses and reduce.
+	expected := len(nodes)
+	done := ctx.Done()
+	for expected > 0 {
+		select {
+		case <-done:
+			return nil, ctx.Err()
+		case resp := <-ch:
+			if resp.err != nil {
+				return nil, errors.Wrap(resp.err, "query fanout")
+			}
+			// if we got a response that we aren't discarding
+			// because it's an error, subtract it from our count...
+			expected -= 1
+
+			// Reduce value.
+
+			result, err = reduceFn(ctx, result, resp.result)
+			if err != nil {
+				cancel()
+				return nil, err
+			}
+		}
+	}
+	// note the deferred Wait above which might override this nil.
+	return result, nil
+}
+
+func (e *ExecutionPlanner) mapper(ctx context.Context, eg *errgroup.Group, ch chan mapResponse, nodes []pilosa.ClusterNode, op types.PlanOperator, reduceFn reduceFunc) (reterr error) {
+	done := ctx.Done()
+	// Execute each node in a separate goroutine.
+	for _, node := range nodes {
+		node := node
+		eg.Go(func() error {
+
+			resp := mapResponse{node: node}
+
+			// Send local shards to mapper, otherwise remote exec.
+			if node.ID == e.systemAPI.NodeID() {
+				iter, err := op.Iterator(ctx, nil)
+				if err != nil {
+					resp.result = nil
+					resp.err = err
+				}
+				row, err := iter.Next(ctx)
+				if err != nil && err != types.ErrNoMoreRows {
+					resp.result = nil
+					resp.err = err
+				}
+				if err != types.ErrNoMoreRows {
+					for {
+						resp.result = append(resp.result, row)
+						row, err = iter.Next(ctx)
+						if err != nil && err != types.ErrNoMoreRows {
+							resp.result = nil
+							resp.err = err
+						}
+						if err == types.ErrNoMoreRows {
+							break
+						}
+					}
+				}
+			} else {
+				results, err := e.remotePlanExec(ctx, node.URI, op)
+				resp.result = results
+				resp.err = err
+			}
+
+			// Return response to the channel.
+			select {
+			case <-done:
+				// If someone just canceled the context
+				// arbitrarily, we could end up here with this
+				// being the first non-nil error handed to
+				// the ErrGroup, in which case, it's the best
+				// explanation we have for why everything's
+				// stopping.
+				return ctx.Err()
+			case ch <- resp:
+				// If we return a non-nil error from this, the
+				// entire errGroup gets canceled. So we don't
+				// want to return a non-nil error if mapReduce
+				// might try to run another mapper against a
+				// different set of nodes. Note that this shouldn't
+				// matter; we just sent the error to mapReduce
+				// anyway, so it probably cancels the ErrGroup
+				// too.
+				if resp.err != nil {
+					return resp.err
+				}
+			}
+			return nil
+		})
+		if reterr != nil {
+			return reterr // exit early if error occurs when running serially
+		}
+	}
+	return nil
+}
+
+func (e *ExecutionPlanner) remotePlanExec(ctx context.Context, addr string, op types.PlanOperator) (types.Rows, error) {
+	b, err := writeOp(op)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create HTTP request.
+	u := fmt.Sprintf("%s/sql", addr)
+	req, err := http.NewRequest("POST", u, bytes.NewReader(b))
+	if err != nil {
+		return nil, errors.Wrap(err, "creating request")
+	}
+
+	// TODO (pok) internal auth
+	//AddAuthToken(ctx, &req.Header)
+
+	req.Header.Set("Content-Length", strconv.Itoa(len(b)))
+	req.Header.Set("Content-Type", "application/octet-stream")
+	req.Header.Set("Accept", "application/octet-stream")
+	req.Header.Set("User-Agent", "pilosa/"+e.systemAPI.Version())
+	req.Header.Set("X-FeatureBase-Plan-Operator", fmt.Sprintf("%T", op))
+
+	// Execute request against the host.
+	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, sql3.NewErrInternalf("error posting internally: %s", err.Error())
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		// we have an error
+		return nil, sql3.NewErrInternalf("error posting internally: %d", resp.StatusCode)
+	}
+
+	var rows types.Rows
+	parser := newWireProtocolParser(e, resp.Body)
+	state := 1
+	for state <= 2 {
+		msg, err := parser.nextMessage()
+		if err != nil {
+			return nil, err
+		}
+		switch state {
+		case 1:
+			switch m := msg.(type) {
+			case *messageSchemaInfo:
+				parser.schema = m.schema
+				state = 2
+
+			case *messageError:
+				return nil, m.err
+
+			default:
+				return nil, sql3.NewErrInternalf("unexpected token %d", msg.Token())
+			}
+
+		case 2:
+			switch m := msg.(type) {
+			case *messageRow:
+				rows = append(rows, m.row)
+
+			case *messageDone:
+				// we're done
+				state = 3
+
+			case *messageError:
+				return nil, m.err
+
+			default:
+				return nil, sql3.NewErrInternalf("unexpected token %d", msg.Token())
+			}
+
+		}
+
+	}
+	return rows, nil
+}
--- a/sql3/planner/opbulkinsert.go
+++ b/sql3/planner/opbulkinsert.go
@ -367,6 +367,8 @@ func (i *bulkInsertCSVRowIter) Next(ctx context.Context) (types.Row, error) {
 				return nil, err
 			}
 			i.currentBatch = nil
+			// update the counter for bulk insert batches
+			pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1)
 		}
 		if i.options.rowsLimit > 0 && i.linesRead >= i.options.rowsLimit {
 			break
@ -378,6 +380,8 @@ func (i *bulkInsertCSVRowIter) Next(ctx context.Context) (types.Row, error) {
 			return nil, err
 		}
 		i.currentBatch = nil
+		// update the counter for bulk insert batches
+		pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1)
 	}
 	return nil, types.ErrNoMoreRows
 }
@ -777,6 +781,8 @@ func (i *bulkInsertNDJsonRowIter) Next(ctx context.Context) (types.Row, error) {
 				return nil, err
 			}
 			i.currentBatch = nil
+			// update the counter for bulk insert batches
+			pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1)
 		}
 		if i.options.rowsLimit > 0 && i.linesRead >= i.options.rowsLimit {
 			break
@ -788,6 +794,8 @@ func (i *bulkInsertNDJsonRowIter) Next(ctx context.Context) (types.Row, error) {
 			return nil, err
 		}
 		i.currentBatch = nil
+		// update the counter for bulk insert batches
+		pilosa.PerfCounterSQLBulkInsertBatchesSec.Add(1)
 	}
 	return nil, types.ErrNoMoreRows
 }
@ -928,5 +936,9 @@ func processBatch(ctx context.Context, planner *ExecutionPlanner, tableName stri
 	if err != nil && err != types.ErrNoMoreRows {
 		return err
 	}
+
+	// update the counter for bulk inserts
+	pilosa.PerfCounterSQLBulkInsertsSec.Add(int64(len(insertValues)))
+
 	return nil
 }
--- a/sql3/planner/opfanout.go
+++ b/sql3/planner/opfanout.go
@ -0,0 +1,114 @@
+// Copyright 2022 Molecula Corp. All rights reserved.
+
+package planner
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/featurebasedb/featurebase/v3/sql3"
+	"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
+)
+
+// PlanOpFanout is a query fanout operator that will execute an operator across all cluster nodes
+type PlanOpFanout struct {
+	planner  *ExecutionPlanner
+	ChildOp  types.PlanOperator
+	warnings []string
+}
+
+func NewPlanOpFanout(planner *ExecutionPlanner, child types.PlanOperator) *PlanOpFanout {
+	return &PlanOpFanout{
+		planner:  planner,
+		ChildOp:  child,
+		warnings: make([]string, 0),
+	}
+}
+
+func (p *PlanOpFanout) Schema() types.Schema {
+	return p.ChildOp.Schema()
+}
+
+func (p *PlanOpFanout) Iterator(ctx context.Context, row types.Row) (types.RowIterator, error) {
+	return newFanOutIterator(p.planner, p.ChildOp), nil
+}
+
+func (p *PlanOpFanout) WithChildren(children ...types.PlanOperator) (types.PlanOperator, error) {
+	if len(children) != 1 {
+		return nil, sql3.NewErrInternalf("unexpected number of children '%d'", len(children))
+	}
+	return NewPlanOpFanout(p.planner, children[0]), nil
+}
+
+func (p *PlanOpFanout) Children() []types.PlanOperator {
+	return []types.PlanOperator{
+		p.ChildOp,
+	}
+}
+
+func (p *PlanOpFanout) Plan() map[string]interface{} {
+	result := make(map[string]interface{})
+	result["_op"] = fmt.Sprintf("%T", p)
+	ps := make([]string, 0)
+	for _, e := range p.Schema() {
+		ps = append(ps, fmt.Sprintf("'%s', '%s', '%s'", e.ColumnName, e.RelationName, e.Type.TypeDescription()))
+	}
+	result["_schema"] = ps
+	result["child"] = p.ChildOp.Plan()
+	return result
+}
+
+func (p *PlanOpFanout) String() string {
+	return ""
+}
+
+func (p *PlanOpFanout) AddWarning(warning string) {
+	p.warnings = append(p.warnings, warning)
+}
+
+func (p *PlanOpFanout) Warnings() []string {
+	return p.warnings
+}
+
+func (p *PlanOpFanout) Expressions() []types.PlanExpression {
+	return []types.PlanExpression{}
+}
+
+func (p *PlanOpFanout) WithUpdatedExpressions(exprs ...types.PlanExpression) (types.PlanOperator, error) {
+	if len(exprs) != 1 {
+		return nil, sql3.NewErrInternalf("unexpected number of exprs '%d'", len(exprs))
+	}
+	return NewPlanOpFilter(p.planner, exprs[0], p.ChildOp), nil
+}
+
+type fanOutIterator struct {
+	planner *ExecutionPlanner
+	childOp types.PlanOperator
+	rows    types.Rows
+}
+
+func newFanOutIterator(planner *ExecutionPlanner, childOp types.PlanOperator) *fanOutIterator {
+	return &fanOutIterator{
+		planner: planner,
+		childOp: childOp,
+	}
+}
+
+func (i *fanOutIterator) Next(ctx context.Context) (types.Row, error) {
+	if i.rows == nil {
+		rows, err := i.planner.mapReducePlanOp(ctx, i.childOp, func(ctx context.Context, prev, v types.Rows) (types.Rows, error) {
+			return append(prev, v...), nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		i.rows = rows
+	}
+	if len(i.rows) > 0 {
+		row := i.rows[0]
+		// Move to next result element.
+		i.rows = i.rows[1:]
+		return row, nil
+	}
+	return nil, types.ErrNoMoreRows
+}
--- a/sql3/planner/opinsert.go
+++ b/sql3/planner/opinsert.go
@ -368,5 +368,8 @@ func (i *insertRowIter) Next(ctx context.Context) (types.Row, error) {
 		return nil, errors.Wrap(err, "importing batch")
 	}

+	// update the counter for inserts
+	pilosa.PerfCounterSQLInsertsSec.Add(int64(batch.Len()))
+
 	return nil, types.ErrNoMoreRows
 }
--- a/sql3/planner/opsystemtable.go
+++ b/sql3/planner/opsystemtable.go
@ -17,16 +17,18 @@ import (
 // exclude this file from SonarCloud dupe eval

 const (
-	fbClusterInfo  = "fb_cluster_info"
-	fbClusterNodes = "fb_cluster_nodes"
-	fbExecRequests = "fb_exec_requests"
+	fbClusterInfo         = "fb_cluster_info"
+	fbClusterNodes        = "fb_cluster_nodes"
+	fbExecRequests        = "fb_exec_requests"
+	fbPerformanceCounters = "fb_performance_counters"

 	fbTableDDL = "fb_table_ddl"
 )

 type systemTable struct {
-	name   string
-	schema types.Schema
+	name           string
+	schema         types.Schema
+	requiresFanout bool
 }

 var systemTables = map[string]*systemTable{
@ -74,6 +76,7 @@ var systemTables = map[string]*systemTable{
 				Type:         parser.NewDataTypeInt(),
 			},
 		},
+		requiresFanout: false,
 	},
 	fbClusterNodes: {
 		name: fbClusterNodes,
@ -109,11 +112,17 @@ var systemTables = map[string]*systemTable{
 				Type:         parser.NewDataTypeBool(),
 			},
 		},
+		requiresFanout: false,
 	},

 	fbExecRequests: {
 		name: fbExecRequests,
 		schema: types.Schema{
+			&types.PlannerColumn{
+				RelationName: fbPerformanceCounters,
+				ColumnName:   "nodeid",
+				Type:         parser.NewDataTypeString(),
+			},
 			&types.PlannerColumn{
 				RelationName: fbExecRequests,
 				ColumnName:   "request_id",
@ -195,6 +204,7 @@ var systemTables = map[string]*systemTable{
 				Type:         parser.NewDataTypeString(),
 			},
 		},
+		requiresFanout: true,
 	},

 	fbTableDDL: {
@ -216,6 +226,44 @@ var systemTables = map[string]*systemTable{
 				Type:         parser.NewDataTypeString(),
 			},
 		},
+		requiresFanout: false,
+	},
+
+	fbPerformanceCounters: {
+		name: fbPerformanceCounters,
+		schema: types.Schema{
+			&types.PlannerColumn{
+				RelationName: fbPerformanceCounters,
+				ColumnName:   "nodeid",
+				Type:         parser.NewDataTypeString(),
+			},
+			&types.PlannerColumn{
+				RelationName: fbPerformanceCounters,
+				ColumnName:   "namespace",
+				Type:         parser.NewDataTypeString(),
+			},
+			&types.PlannerColumn{
+				RelationName: fbPerformanceCounters,
+				ColumnName:   "subsystem",
+				Type:         parser.NewDataTypeString(),
+			},
+			&types.PlannerColumn{
+				RelationName: fbPerformanceCounters,
+				ColumnName:   "counter_name",
+				Type:         parser.NewDataTypeString(),
+			},
+			&types.PlannerColumn{
+				RelationName: fbPerformanceCounters,
+				ColumnName:   "value",
+				Type:         parser.NewDataTypeInt(),
+			},
+			&types.PlannerColumn{
+				RelationName: fbPerformanceCounters,
+				ColumnName:   "counter_type",
+				Type:         parser.NewDataTypeInt(),
+			},
+		},
+		requiresFanout: true,
 	},
 }

@ -279,6 +327,10 @@ func (p *PlanOpSystemTable) Iterator(ctx context.Context, row types.Row) (types.
 		return &fbTableDDLRowIter{
 			planner: p.planner,
 		}, nil
+	case fbPerformanceCounters:
+		return &fbPerformanceCountersRowIter{
+			planner: p.planner,
+		}, nil
 	default:
 		return nil, sql3.NewErrInternalf("unable to find system table '%s'", p.table.name)
 	}
@ -367,9 +419,11 @@ func (i *fbExecRequestsRowIter) Next(ctx context.Context) (types.Row, error) {
 		}
 	}

+	nodeId := i.planner.systemAPI.NodeID()
 	if len(i.result) > 0 {
 		n := i.result[0]
 		row := []interface{}{
+			nodeId,
 			n.RequestID,
 			n.UserID,
 			n.StartTime,
@ -533,3 +587,37 @@ func (i *fbTableDDLRowIter) Next(ctx context.Context) (types.Row, error) {
 	}
 	return nil, types.ErrNoMoreRows
 }
+
+type fbPerformanceCountersRowIter struct {
+	planner *ExecutionPlanner
+	result  []pilosa.PerformanceCounter
+}
+
+var _ types.RowIterator = (*fbPerformanceCountersRowIter)(nil)
+
+func (i *fbPerformanceCountersRowIter) Next(ctx context.Context) (types.Row, error) {
+	if i.result == nil {
+		var err error
+		i.result, err = pilosa.PerfCounters.ListCounters()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	nodeId := i.planner.systemAPI.NodeID()
+	if len(i.result) > 0 {
+		n := i.result[0]
+		row := []interface{}{
+			nodeId,
+			n.NameSpace,
+			n.SubSystem,
+			n.CounterName,
+			n.Value,
+			n.CounterType,
+		}
+		// Move to next result element.
+		i.result = i.result[1:]
+		return row, nil
+	}
+	return nil, types.ErrNoMoreRows
+}
--- a/sql3/planner/types/operator.go
+++ b/sql3/planner/types/operator.go
@ -80,9 +80,12 @@ func (r Schema) Plan() []map[string]interface{} {
 	return result
 }

-// Row is a tuple of values
+// Row is a tuple (of values)
 type Row []interface{}

+// Rows is a table of rows
+type Rows []Row
+
 // Append appends all the values in r2 to this row and returns the result
 func (r Row) Append(r2 Row) Row {
 	row := make(Row, len(r)+len(r2))
--- a/sql3/planner/wireprotocol.go
+++ b/sql3/planner/wireprotocol.go
@ -0,0 +1,211 @@
+package planner
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"io"
+
+	"github.com/featurebasedb/featurebase/v3/sql3"
+	"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
+	"github.com/featurebasedb/featurebase/v3/wireprotocol"
+)
+
+type wireProtocolMessage interface {
+	Token() int16
+}
+
+func writeOp(op types.PlanOperator) ([]byte, error) {
+	buf := new(bytes.Buffer)
+	writer := bufio.NewWriter(buf)
+	// serialize a plan op - for now we are just supporting
+	// system tables, and we'll send the name of the system table
+	switch op := op.(type) {
+	case *PlanOpSystemTable:
+		// write token
+		b := make([]byte, 2)
+		binary.BigEndian.PutUint16(b, uint16(wireprotocol.TOKEN_PLAN_OP))
+		writer.Write(b)
+		// write table name len
+		t := op.table.name
+		b = make([]byte, 4)
+		binary.BigEndian.PutUint32(b, uint32(len(t)))
+		writer.Write(b)
+		// write table name
+		writer.WriteString(op.table.name)
+		writer.Flush()
+	default:
+		return []byte{}, sql3.NewErrInternalf("unexpected plan operator type '%T'", op)
+	}
+	return buf.Bytes(), nil
+}
+
+type wireProtocolParser struct {
+	planner *ExecutionPlanner
+	reader  io.Reader
+	schema  types.Schema
+}
+
+func newWireProtocolParser(p *ExecutionPlanner, reader io.Reader) *wireProtocolParser {
+	return &wireProtocolParser{
+		planner: p,
+		reader:  reader,
+	}
+}
+
+func (f *wireProtocolParser) nextMessage() (wireProtocolMessage, error) {
+	var t int16
+	err := binary.Read(f.reader, binary.BigEndian, &t)
+	if err != nil {
+		return nil, err
+	}
+	switch t {
+	case wireprotocol.TOKEN_SCHEMA_INFO:
+		return newMessageSchemaInfo(f.planner, f.reader)
+
+	case wireprotocol.TOKEN_ROW:
+		if f.schema == nil {
+			return nil, sql3.NewErrInternalf("schema uninitialized")
+		}
+		return newMessageRow(f.planner, f.reader, f.schema)
+
+	case wireprotocol.TOKEN_ERROR_MESSAGE:
+		return newMessageError(f.planner, f.reader)
+
+	case wireprotocol.TOKEN_DONE:
+		return newMessageDone(f.planner, f.reader)
+
+	case wireprotocol.TOKEN_PLAN_OP:
+		return newMessagePlanOp(f.planner, f.reader)
+
+	default:
+		return nil, sql3.NewErrInternalf("unexpected token %d", t)
+	}
+}
+
+type messagePlanOp struct {
+	token int16
+	op    types.PlanOperator
+}
+
+var _ wireProtocolMessage = (*messagePlanOp)(nil)
+
+func newMessagePlanOp(p *ExecutionPlanner, reader io.Reader) (*messagePlanOp, error) {
+
+	var len int32
+	err := binary.Read(reader, binary.BigEndian, &len)
+	if err != nil {
+		return nil, err
+	}
+	bname := make([]byte, len)
+	err = binary.Read(reader, binary.BigEndian, &bname)
+	if err != nil {
+		return nil, err
+	}
+	name := string(bname)
+
+	st, ok := systemTables[name]
+	if !ok {
+		return nil, sql3.NewErrInternalf("unexpected system table name %s", name)
+	}
+
+	return &messagePlanOp{
+		token: wireprotocol.TOKEN_PLAN_OP,
+		op:    NewPlanOpSystemTable(p, st),
+	}, nil
+}
+
+func (m *messagePlanOp) Token() int16 {
+	return m.token
+}
+
+type messageError struct {
+	token int16
+	err   error
+}
+
+var _ wireProtocolMessage = (*messageError)(nil)
+
+func newMessageError(p *ExecutionPlanner, reader io.Reader) (*messageError, error) {
+	var len int32
+	err := binary.Read(reader, binary.BigEndian, &len)
+	if err != nil {
+		return nil, err
+	}
+	bname := make([]byte, len)
+	err = binary.Read(reader, binary.BigEndian, &bname)
+	if err != nil {
+		return nil, err
+	}
+	errMsg := string(bname)
+
+	return &messageError{
+		token: wireprotocol.TOKEN_ERROR_MESSAGE,
+		err:   errors.New(errMsg),
+	}, nil
+}
+
+func (m *messageError) Token() int16 {
+	return m.token
+}
+
+type messageSchemaInfo struct {
+	token  int16
+	schema types.Schema
+}
+
+var _ wireProtocolMessage = (*messageSchemaInfo)(nil)
+
+func newMessageSchemaInfo(p *ExecutionPlanner, reader io.Reader) (*messageSchemaInfo, error) {
+	schema, err := wireprotocol.ReadSchema(reader)
+	if err != nil {
+		return nil, err
+	}
+	return &messageSchemaInfo{
+		token:  wireprotocol.TOKEN_SCHEMA_INFO,
+		schema: schema,
+	}, nil
+}
+
+func (m *messageSchemaInfo) Token() int16 {
+	return m.token
+}
+
+type messageRow struct {
+	token int16
+	row   types.Row
+}
+
+var _ wireProtocolMessage = (*messageRow)(nil)
+
+func newMessageRow(p *ExecutionPlanner, reader io.Reader, schema types.Schema) (*messageRow, error) {
+	row, err := wireprotocol.ReadRow(reader, schema)
+	if err != nil {
+		return nil, err
+	}
+	return &messageRow{
+		token: wireprotocol.TOKEN_ROW,
+		row:   row,
+	}, nil
+}
+
+func (m *messageRow) Token() int16 {
+	return m.token
+}
+
+type messageDone struct {
+	token int16
+}
+
+var _ wireProtocolMessage = (*messageDone)(nil)
+
+func newMessageDone(p *ExecutionPlanner, reader io.Reader) (*messageDone, error) {
+	return &messageDone{
+		token: wireprotocol.TOKEN_DONE,
+	}, nil
+}
+
+func (m *messageDone) Token() int16 {
+	return m.token
+}
--- a/sql3/sql_complex_test.go
+++ b/sql3/sql_complex_test.go
@ -36,6 +36,92 @@ func TestPlanner_Misc(t *testing.T) {
 	assert.True(t, d.EqualTo(pql.NewDecimal(12345678, 6)))
 }

+func TestPlanner_SystemTableFanout(t *testing.T) {
+	c := test.MustRunCluster(t, 3)
+	defer c.Close()
+
+	server := c.GetNode(0).Server
+
+	t.Run("PerfCounters", func(t *testing.T) {
+		results, columns, err := sql_test.MustQueryRows(t, server, `select * from fb_performance_counters`)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(results) != 15 {
+			t.Fatal(fmt.Errorf("unexpected result set length"))
+		}
+
+		if diff := cmp.Diff([]*pilosa.WireQueryField{
+			wireQueryFieldString("nodeid"),
+			wireQueryFieldString("namespace"),
+			wireQueryFieldString("subsystem"),
+			wireQueryFieldString("counter_name"),
+			wireQueryFieldInt("value"),
+			wireQueryFieldInt("counter_type"),
+		}, columns); diff != "" {
+			t.Fatal(diff)
+		}
+	})
+
+	t.Run("SystemTablesExecRequests", func(t *testing.T) {
+		results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select * from fb_exec_requests`)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if len(results) != 2 {
+			t.Fatal(fmt.Errorf("unexpected result set length"))
+		}
+
+		if diff := cmp.Diff([]*pilosa.WireQueryField{
+			wireQueryFieldString("nodeid"),
+			wireQueryFieldString("request_id"),
+			wireQueryFieldString("user"),
+			wireQueryFieldTimestamp("start_time"),
+			wireQueryFieldTimestamp("end_time"),
+			wireQueryFieldString("status"),
+			wireQueryFieldString("wait_type"),
+			wireQueryFieldInt("wait_time"),
+			wireQueryFieldString("wait_resource"),
+			wireQueryFieldInt("cpu_time"),
+			wireQueryFieldInt("elapsed_time"),
+			wireQueryFieldInt("reads"),
+			wireQueryFieldInt("writes"),
+			wireQueryFieldInt("logical_reads"),
+			wireQueryFieldInt("row_count"),
+			wireQueryFieldString("sql"),
+			wireQueryFieldString("plan"),
+		}, columns); diff != "" {
+			t.Fatal(diff)
+		}
+	})
+
+	t.Run("SystemTablesExecRequestsAgg", func(t *testing.T) {
+		_, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select 
+		count(request_id) as request_count,
+		min(elapsed_time) as min_duration,
+		max(elapsed_time) as max_duration,
+		avg(elapsed_time) as avg_duration
+	from 
+		fb_exec_requests 
+	where 
+		status = 'complete';`)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if diff := cmp.Diff([]*pilosa.WireQueryField{
+			wireQueryFieldInt("request_count"),
+			wireQueryFieldInt("min_duration"),
+			wireQueryFieldInt("max_duration"),
+			wireQueryFieldDecimal("avg_duration", 4),
+		}, columns); diff != "" {
+			t.Fatal(diff)
+		}
+	})
+
+}
+
 func TestPlanner_Show(t *testing.T) {
 	c := test.MustRunCluster(t, 1)
 	defer c.Close()
@ -102,64 +188,12 @@ func TestPlanner_Show(t *testing.T) {
 		}
 	})

-	t.Run("SystemTablesExecRequests", func(t *testing.T) {
-		_, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select * from fb_exec_requests`)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if diff := cmp.Diff([]*pilosa.WireQueryField{
-			wireQueryFieldString("request_id"),
-			wireQueryFieldString("user"),
-			wireQueryFieldTimestamp("start_time"),
-			wireQueryFieldTimestamp("end_time"),
-			wireQueryFieldString("status"),
-			wireQueryFieldString("wait_type"),
-			wireQueryFieldInt("wait_time"),
-			wireQueryFieldString("wait_resource"),
-			wireQueryFieldInt("cpu_time"),
-			wireQueryFieldInt("elapsed_time"),
-			wireQueryFieldInt("reads"),
-			wireQueryFieldInt("writes"),
-			wireQueryFieldInt("logical_reads"),
-			wireQueryFieldInt("row_count"),
-			wireQueryFieldString("sql"),
-			wireQueryFieldString("plan"),
-		}, columns); diff != "" {
-			t.Fatal(diff)
-		}
-	})
-
-	t.Run("SystemTablesExecRequestsAgg", func(t *testing.T) {
-		_, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `select 
-		count(request_id) as request_count,
-		min(elapsed_time) as min_duration,
-		max(elapsed_time) as max_duration,
-		avg(elapsed_time) as avg_duration
-	from 
-		fb_exec_requests 
-	where 
-		status = 'complete';`)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if diff := cmp.Diff([]*pilosa.WireQueryField{
-			wireQueryFieldInt("request_count"),
-			wireQueryFieldInt("min_duration"),
-			wireQueryFieldInt("max_duration"),
-			wireQueryFieldDecimal("avg_duration", 4),
-		}, columns); diff != "" {
-			t.Fatal(diff)
-		}
-	})
-
 	t.Run("ShowTables", func(t *testing.T) {
 		results, columns, err := sql_test.MustQueryRows(t, c.GetNode(0).Server, `SHOW TABLES`)
 		if err != nil {
 			t.Fatal(err)
 		}
-		if len(results) != 6 {
+		if len(results) != 7 {
 			t.Fatal(fmt.Errorf("unexpected result set length"))
 		}

--- a/stats/stats.go
+++ b/stats/stats.go
@ -1,274 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package stats
-
-import (
-	"expvar"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/featurebasedb/featurebase/v3/logger"
-)
-
-// Expvar global expvar map.
-var Expvar *expvar.Map
-
-// StatsClient represents a client to a stats server.
-type StatsClient interface {
-	// Returns a sorted list of tags on the client.
-	Tags() []string
-
-	// Returns a new client with additional tags appended.
-	WithTags(tags ...string) StatsClient
-
-	// Tracks the number of times something occurs per second.
-	Count(name string, value int64, rate float64)
-
-	// Tracks the number of times something occurs per second with custom tags
-	CountWithCustomTags(name string, value int64, rate float64, tags []string)
-
-	// Sets the value of a metric.
-	Gauge(name string, value float64, rate float64)
-
-	// Tracks statistical distribution of a metric.
-	Histogram(name string, value float64, rate float64)
-
-	// Tracks number of unique elements.
-	Set(name string, value string, rate float64)
-
-	// Tracks timing information for a metric.
-	Timing(name string, value time.Duration, rate float64)
-
-	// SetLogger Set the logger output type
-	SetLogger(logger logger.Logger)
-
-	// Starts the service
-	Open()
-
-	// Closes the client
-	Close() error
-}
-
-// NopStatsClient represents a client that doesn't do anything.
-var NopStatsClient StatsClient = &nopStatsClient{}
-
-type nopStatsClient struct{}
-
-func (c *nopStatsClient) Tags() []string                                                            { return nil }
-func (c *nopStatsClient) WithTags(tags ...string) StatsClient                                       { return c }
-func (c *nopStatsClient) Count(name string, value int64, rate float64)                              {}
-func (c *nopStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) {}
-func (c *nopStatsClient) Gauge(name string, value float64, rate float64)                            {}
-func (c *nopStatsClient) Histogram(name string, value float64, rate float64)                        {}
-func (c *nopStatsClient) Set(name string, value string, rate float64)                               {}
-func (c *nopStatsClient) Timing(name string, value time.Duration, rate float64)                     {}
-func (c *nopStatsClient) SetLogger(logger logger.Logger)                                            {}
-func (c *nopStatsClient) Open()                                                                     {}
-func (c *nopStatsClient) Close() error                                                              { return nil }
-
-// expvarStatsClient writes stats out to expvars.
-type expvarStatsClient struct {
-	mu   sync.Mutex
-	m    *expvar.Map
-	tags []string
-}
-
-// NewExpvarStatsClient returns a new instance of ExpvarStatsClient.
-// This client points at the root of the expvar index map.
-func NewExpvarStatsClient() *expvarStatsClient {
-	if Expvar == nil {
-		Expvar = expvar.NewMap("index")
-	}
-	return &expvarStatsClient{
-		m: Expvar,
-	}
-}
-
-// Tags returns a sorted list of tags on the client.
-func (c *expvarStatsClient) Tags() []string {
-	return nil
-}
-
-// WithTags returns a new client with additional tags appended.
-func (c *expvarStatsClient) WithTags(tags ...string) StatsClient {
-	m := &expvar.Map{}
-	m.Init()
-	c.m.Set(strings.Join(tags, ","), m)
-
-	return &expvarStatsClient{
-		m:    m,
-		tags: unionStringSlice(c.tags, tags),
-	}
-}
-
-// Count tracks the number of times something occurs.
-func (c *expvarStatsClient) Count(name string, value int64, rate float64) {
-	c.m.Add(name, value)
-}
-
-// CountWithCustomTags Tracks the number of times something occurs per second with custom tags
-func (c *expvarStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) {
-	c.m.Add(name, value)
-}
-
-// Gauge sets the value of a metric.
-func (c *expvarStatsClient) Gauge(name string, value float64, rate float64) {
-	var f expvar.Float
-	f.Set(value)
-	c.m.Set(name, &f)
-}
-
-// Histogram tracks statistical distribution of a metric.
-// This works the same as gauge for this client.
-func (c *expvarStatsClient) Histogram(name string, value float64, rate float64) {
-	c.Gauge(name, value, rate)
-}
-
-// Set tracks number of unique elements.
-func (c *expvarStatsClient) Set(name string, value string, rate float64) {
-	var s expvar.String
-	s.Set(value)
-	c.m.Set(name, &s)
-}
-
-// Timing tracks timing information for a metric.
-func (c *expvarStatsClient) Timing(name string, value time.Duration, rate float64) {
-	c.mu.Lock()
-	d, _ := c.m.Get(name).(time.Duration)
-	c.m.Set(name, d+value)
-	c.mu.Unlock()
-}
-
-// SetLogger has no logger.
-func (c *expvarStatsClient) SetLogger(logger logger.Logger) {
-}
-
-// Open no-op.
-func (c *expvarStatsClient) Open() {}
-
-// Close no-op.
-func (c *expvarStatsClient) Close() error { return nil }
-
-// MultiStatsClient joins multiple stats clients together.
-type MultiStatsClient []StatsClient
-
-// Tags returns tags from the first client.
-func (a MultiStatsClient) Tags() []string {
-	if len(a) > 0 {
-		return a[0].Tags()
-	}
-	return nil
-}
-
-// WithTags returns a new set of clients with the additional tags.
-func (a MultiStatsClient) WithTags(tags ...string) StatsClient {
-	other := make(MultiStatsClient, len(a))
-	for i := range a {
-		other[i] = a[i].WithTags(tags...)
-	}
-	return other
-}
-
-// Count tracks the number of times something occurs per second on all clients.
-func (a MultiStatsClient) Count(name string, value int64, rate float64) {
-	for _, c := range a {
-		c.Count(name, value, rate)
-	}
-}
-
-// CountWithCustomTags Tracks the number of times something occurs per second with custom tags
-func (a MultiStatsClient) CountWithCustomTags(name string, value int64, rate float64, tags []string) {
-	for _, c := range a {
-		c.CountWithCustomTags(name, value, rate, tags)
-	}
-}
-
-// Gauge sets the value of a metric on all clients.
-func (a MultiStatsClient) Gauge(name string, value float64, rate float64) {
-	for _, c := range a {
-		c.Gauge(name, value, rate)
-	}
-}
-
-// Histogram tracks statistical distribution of a metric on all clients.
-func (a MultiStatsClient) Histogram(name string, value float64, rate float64) {
-	for _, c := range a {
-		c.Histogram(name, value, rate)
-	}
-}
-
-// Set tracks number of unique elements on all clients.
-func (a MultiStatsClient) Set(name string, value string, rate float64) {
-	for _, c := range a {
-		c.Set(name, value, rate)
-	}
-}
-
-// Timing tracks timing information for a metric on all clients.
-func (a MultiStatsClient) Timing(name string, value time.Duration, rate float64) {
-	for _, c := range a {
-		c.Timing(name, value, rate)
-	}
-}
-
-// SetLogger Sets the StatsD logger output type.
-func (a MultiStatsClient) SetLogger(logger logger.Logger) {
-	for _, c := range a {
-		c.SetLogger(logger)
-	}
-}
-
-// Open starts the stat service.
-func (a MultiStatsClient) Open() {
-	for _, c := range a {
-		c.Open()
-	}
-}
-
-// Close shuts down the stats clients.
-func (a MultiStatsClient) Close() error {
-	for _, c := range a {
-		err := c.Close()
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// unionStringSlice returns a sorted set of tags which combine a & b.
-func unionStringSlice(a, b []string) []string {
-	// Sort both sets first.
-	sort.Strings(a)
-	sort.Strings(b)
-
-	// Find size of largest slice.
-	n := len(a)
-	if len(b) > n {
-		n = len(b)
-	}
-
-	// Exit if both sets are empty.
-	if n == 0 {
-		return nil
-	}
-
-	// Iterate over both in order and merge.
-	other := make([]string, 0, n)
-	for len(a) > 0 || len(b) > 0 {
-		if len(a) == 0 {
-			other, b = append(other, b[0]), b[1:]
-		} else if len(b) == 0 {
-			other, a = append(other, a[0]), a[1:]
-		} else if a[0] < b[0] {
-			other, a = append(other, a[0]), a[1:]
-		} else if b[0] < a[0] {
-			other, b = append(other, b[0]), b[1:]
-		} else {
-			other, a, b = append(other, a[0]), a[1:], b[1:]
-		}
-	}
-	return other
-}
--- a/stats/stats_test.go
+++ b/stats/stats_test.go
@ -1,258 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package stats_test
-
-import (
-	"context"
-	"fmt"
-	"net/http/httptest"
-	"strings"
-	"testing"
-	"time"
-
-	pilosa "github.com/featurebasedb/featurebase/v3"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/featurebasedb/featurebase/v3/stats"
-	"github.com/featurebasedb/featurebase/v3/test"
-)
-
-// TestMultiStatClient_Expvar run the multistat client with exp var
-// since the EXPVAR data is stored in a global we should run these in one test function
-func TestMultiStatClient_Expvar(t *testing.T) {
-	hldr := test.MustOpenHolder(t)
-
-	c := stats.NewExpvarStatsClient()
-	ms := make(stats.MultiStatsClient, 1)
-	ms[0] = c
-	hldr.Stats = ms
-
-	hldr.SetBit("d", "f", 0, 0)
-	hldr.SetBit("d", "f", 0, 1)
-	hldr.SetBit("d", "f", 0, pilosa.ShardWidth)
-	hldr.SetBit("d", "f", 0, pilosa.ShardWidth+2)
-	hldr.ClearBit("d", "f", 0, 1)
-
-	indexStats := fmt.Sprintf(`{"%s": %d, "%s": %d}`, pilosa.MetricClearBit, 1, pilosa.MetricSetBit, 4)
-
-	if stats.Expvar.String() != `{"index:d": `+indexStats+`}` {
-		t.Fatalf("unexpected expvar : %s", stats.Expvar.String())
-	}
-
-	hldr.Stats.CountWithCustomTags("cc", 1, 1.0, []string{"foo:bar"})
-	if stats.Expvar.String() != `{"cc": 1, "index:d": `+indexStats+`}` {
-		t.Fatalf("unexpected expvar : %s", stats.Expvar.String())
-	}
-
-	// Gauge creates a unique key, subsequent Gauge calls will overwrite
-	hldr.Stats.Gauge("g", 5, 1.0)
-	hldr.Stats.Gauge("g", 8, 1.0)
-	if stats.Expvar.String() != `{"cc": 1, "g": 8, "index:d": `+indexStats+`}` {
-		t.Fatalf("unexpected expvar : %s", stats.Expvar.String())
-	}
-
-	// Set creates a unique key, subsequent sets will overwrite
-	hldr.Stats.Set("s", "4", 1.0)
-	hldr.Stats.Set("s", "7", 1.0)
-	if stats.Expvar.String() != `{"cc": 1, "g": 8, "index:d": `+indexStats+`, "s": "7"}` {
-		t.Fatalf("unexpected expvar : %s", stats.Expvar.String())
-	}
-
-	// Record timing duration and a uniquely Set key/value
-	dur, _ := time.ParseDuration("123us")
-	hldr.Stats.Timing("tt", dur, 1.0)
-	if stats.Expvar.String() != `{"cc": 1, "g": 8, "index:d": `+indexStats+`, "s": "7", "tt": 123µs}` {
-		t.Fatalf("unexpected expvar : %s", stats.Expvar.String())
-	}
-
-	// Expvar histogram is implemented as a gauge
-	hldr.Stats.Histogram("hh", 3, 1.0)
-	if stats.Expvar.String() != `{"cc": 1, "g": 8, "hh": 3, "index:d": `+indexStats+`, "s": "7", "tt": 123µs}` {
-		t.Fatalf("unexpected expvar : %s", stats.Expvar.String())
-	}
-
-	// Expvar should ignore earlier set tags from setbit
-	if hldr.Stats.Tags() != nil {
-		t.Fatalf("unexpected tag")
-	}
-}
-
-func TestStatsCount_TopN(t *testing.T) {
-	c := test.MustRunCluster(t, 1)
-	defer c.Close()
-	hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
-
-	// Execute query.
-	called := false
-	hldr.Holder.Stats = &MockStats{
-		mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
-			if name != "query_topn_total" {
-				t.Errorf("Expected query_topn_total, Results %s", name)
-			}
-
-			if tags[0] != "index:d" {
-				t.Errorf("Expected index, Results %s", tags[0])
-			}
-
-			called = true
-		},
-	}
-
-	hldr.SetBit("d", "f", 0, 0)
-	hldr.SetBit("d", "f", 0, 1)
-	hldr.SetBit("d", "f", 0, pilosa.ShardWidth)
-	hldr.SetBit("d", "f", 0, pilosa.ShardWidth+2)
-
-	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `TopN(field=f, n=2)`}); err != nil {
-		t.Fatal(err)
-	}
-	if !called {
-		t.Error("CountWithCustomTags name isn't called")
-	}
-}
-
-func TestStatsCount_Bitmap(t *testing.T) {
-	// Cluster has to be unhsared because we're mocking the stats which writes
-	// to a holder in use by other tests.
-	c := test.MustRunUnsharedCluster(t, 1)
-	defer c.Close()
-	hldr := test.Holder{Holder: c.GetNode(0).Server.Holder()}
-	called := false
-	hldr.Holder.Stats = &MockStats{
-		mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
-			if name != pilosa.MetricRow {
-				t.Errorf("Expected %s, Results %s", pilosa.MetricRow, name)
-			}
-
-			if tags[0] != "index:d" {
-				t.Errorf("Expected index, Results %s", tags[0])
-			}
-
-			called = true
-		},
-	}
-
-	hldr.SetBit("d", "f", 0, 0)
-	hldr.SetBit("d", "f", 0, 1)
-
-	if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{Index: "d", Query: `Row(f=0)`}); err != nil {
-		t.Fatal(err)
-	}
-	if !called {
-		t.Error("CountWithCustomTags name isn't called")
-	}
-}
-
-func TestStatsCount_APICalls(t *testing.T) {
-	// We can't share a cluster when we're modifying its stats counter.
-	cluster := test.MustRunUnsharedCluster(t, 1)
-	defer cluster.Close()
-	cmd := cluster.GetNode(0)
-	h := cmd.Handler.(*pilosa.Handler).Handler
-	holder := cmd.Server.Holder()
-	hldr := test.Holder{Holder: holder}
-
-	t.Run("create index", func(t *testing.T) {
-		called := false
-		hldr.Stats = &MockStats{
-			mockCount: func(name string, value int64, rate float64) {
-				if name != pilosa.MetricCreateIndex {
-					t.Errorf("Expected %v, Results %s", pilosa.MetricCreateIndex, name)
-				}
-				called = true
-			},
-		}
-		w := httptest.NewRecorder()
-		h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i", strings.NewReader("")))
-		if !called {
-			t.Error("Count isn't called")
-		}
-	})
-
-	t.Run("create field", func(t *testing.T) {
-		called := false
-		hldr.Stats = &MockStats{
-			mockCountWithTags: func(name string, value int64, rate float64, index []string) {
-				if name != pilosa.MetricCreateField {
-					t.Errorf("Expected %v, Results %s", pilosa.MetricCreateField, name)
-				}
-				if index[0] != "index:i" {
-					t.Errorf("Expected index:i, Results %s", index)
-				}
-
-				called = true
-			},
-		}
-		w := httptest.NewRecorder()
-		h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/field/f", strings.NewReader("")))
-		if !called {
-			t.Error("Count isn't called")
-		}
-	})
-
-	t.Run("delete field", func(t *testing.T) {
-		called := false
-		hldr.Stats = &MockStats{
-			mockCountWithTags: func(name string, value int64, rate float64, index []string) {
-				if name != pilosa.MetricDeleteField {
-					t.Errorf("Expected %v, Results %s", pilosa.MetricDeleteField, name)
-				}
-				if index[0] != "index:i" {
-					t.Errorf("Expected index:i, Results %s", index)
-				}
-
-				called = true
-			},
-		}
-		w := httptest.NewRecorder()
-		h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f", strings.NewReader("")))
-		if !called {
-			t.Error("Count isn't called")
-		}
-	})
-
-	t.Run("delete index", func(t *testing.T) {
-		called := false
-		hldr.Stats = &MockStats{
-			mockCount: func(name string, value int64, rate float64) {
-				if name != pilosa.MetricDeleteIndex {
-					t.Errorf("Expected %v, Results %s", pilosa.MetricDeleteIndex, name)
-				}
-
-				called = true
-			},
-		}
-		w := httptest.NewRecorder()
-		h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader("")))
-		if !called {
-			t.Error("Count isn't called")
-		}
-	})
-
-}
-
-type MockStats struct {
-	mockCount         func(name string, value int64, rate float64)
-	mockCountWithTags func(name string, value int64, rate float64, tags []string)
-}
-
-func (s *MockStats) Count(name string, value int64, rate float64) {
-	if s.mockCount != nil {
-		s.mockCount(name, value, rate)
-	}
-}
-
-func (s *MockStats) CountWithCustomTags(name string, value int64, rate float64, tags []string) {
-	if s.mockCountWithTags != nil {
-		s.mockCountWithTags(name, value, rate, tags)
-	}
-}
-
-func (c *MockStats) Tags() []string                                        { return nil }
-func (c *MockStats) WithTags(tags ...string) stats.StatsClient             { return c }
-func (c *MockStats) Gauge(name string, value float64, rate float64)        {}
-func (c *MockStats) Histogram(name string, value float64, rate float64)    {}
-func (c *MockStats) Set(name string, value string, rate float64)           {}
-func (c *MockStats) Timing(name string, value time.Duration, rate float64) {}
-func (c *MockStats) SetLogger(logger logger.Logger)                        {}
-func (c *MockStats) Open()                                                 {}
-func (c *MockStats) Close() error                                          { return nil }
--- a/statsd/statsd.go
+++ b/statsd/statsd.go
@ -1,152 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package statsd
-
-import (
-	"sort"
-	"time"
-
-	"github.com/DataDog/datadog-go/statsd"
-	"github.com/featurebasedb/featurebase/v3/logger"
-	"github.com/featurebasedb/featurebase/v3/stats"
-)
-
-// StatsD protocol wrapper using the DataDog library that added Tags to the StatsD protocol
-// statsD defailt host is "127.0.0.1:8125"
-
-const (
-	// bufferLen Stats lient buffer size.
-	bufferLen = 1024
-)
-
-// Ensure client implements interface.
-var _ stats.StatsClient = &statsClient{}
-
-// statsClient represents a StatsD implementation of pilosa.statsClient.
-type statsClient struct {
-	client *statsd.Client
-	tags   []string
-	logger logger.Logger
-
-	// prefix is appended to each metric event name
-	prefix string
-}
-
-// NewStatsClient returns a new instance of StatsClient.
-func NewStatsClient(host string, namespace string) (*statsClient, error) {
-	c, err := statsd.NewBuffered(host, bufferLen)
-	if err != nil {
-		return nil, err
-	}
-
-	return &statsClient{
-		client: c,
-		logger: logger.NopLogger,
-		prefix: namespace + ".",
-	}, nil
-}
-
-// Open no-op
-func (c *statsClient) Open() {}
-
-// Close closes the connection to the agent.
-func (c *statsClient) Close() error {
-	return c.client.Close()
-}
-
-// Tags returns a sorted list of tags on the client.
-func (c *statsClient) Tags() []string {
-	return c.tags
-}
-
-// WithTags returns a new client with additional tags appended.
-func (c *statsClient) WithTags(tags ...string) stats.StatsClient {
-	return &statsClient{
-		client: c.client,
-		tags:   unionStringSlice(c.tags, tags),
-		logger: c.logger,
-	}
-}
-
-// Count tracks the number of times something occurs per second.
-func (c *statsClient) Count(name string, value int64, rate float64) {
-	if err := c.client.Count(c.prefix+name, value, c.tags, rate); err != nil {
-		c.logger.Errorf("statsd.StatsClient.Count error: %s", err)
-	}
-}
-
-// CountWithCustomTags tracks the number of times something occurs per second with custom tags.
-func (c *statsClient) CountWithCustomTags(name string, value int64, rate float64, t []string) {
-	tags := append(c.tags, t...)
-	if err := c.client.Count(c.prefix+name, value, tags, rate); err != nil {
-		c.logger.Errorf("statsd.StatsClient.Count error: %s", err)
-	}
-}
-
-// Gauge sets the value of a metric.
-func (c *statsClient) Gauge(name string, value float64, rate float64) {
-	if err := c.client.Gauge(c.prefix+name, value, c.tags, rate); err != nil {
-		c.logger.Errorf("statsd.StatsClient.Gauge error: %s", err)
-	}
-}
-
-// Histogram tracks statistical distribution of a metric.
-func (c *statsClient) Histogram(name string, value float64, rate float64) {
-	if err := c.client.Histogram(c.prefix+name, value, c.tags, rate); err != nil {
-		c.logger.Errorf("statsd.StatsClient.Histogram error: %s", err)
-	}
-}
-
-// Set tracks number of unique elements.
-func (c *statsClient) Set(name string, value string, rate float64) {
-	if err := c.client.Set(c.prefix+name, value, c.tags, rate); err != nil {
-		c.logger.Errorf("statsd.StatsClient.Set error: %s", err)
-	}
-}
-
-// Timing tracks timing information for a metric.
-func (c *statsClient) Timing(name string, value time.Duration, rate float64) {
-	if err := c.client.Timing(c.prefix+name, value, c.tags, rate); err != nil {
-		c.logger.Errorf("statsd.StatsClient.Timing error: %s", err)
-	}
-}
-
-// SetLogger sets the logger for client.
-func (c *statsClient) SetLogger(logger logger.Logger) {
-	c.logger = logger
-}
-
-// unionStringSlice returns a sorted set of tags which combine a & b.
-func unionStringSlice(a, b []string) []string {
-	// Sort both sets first.
-	sort.Strings(a)
-	sort.Strings(b)
-
-	// Find size of largest slice.
-	n := len(a)
-	if len(b) > n {
-		n = len(b)
-	}
-
-	// Exit if both sets are empty.
-	if n == 0 {
-		return nil
-	}
-
-	// Iterate over both in order and merge.
-	other := make([]string, 0, n)
-	for len(a) > 0 || len(b) > 0 {
-		if len(a) == 0 {
-			other, b = append(other, b[0]), b[1:]
-		} else if len(b) == 0 {
-			other, a = append(other, a[0]), a[1:]
-		} else if a[0] < b[0] {
-			other, a = append(other, a[0]), a[1:]
-		} else if b[0] < a[0] {
-			other, b = append(other, b[0]), b[1:]
-		} else {
-			other, a, b = append(other, a[0]), a[1:], b[1:]
-		}
-	}
-	return other
-}
--- a/statsd/statsd_test.go
+++ b/statsd/statsd_test.go
@ -1,50 +0,0 @@
-// Copyright 2022 Molecula Corp. (DBA FeatureBase).
-// SPDX-License-Identifier: Apache-2.0
-package statsd_test
-
-import (
-	"reflect"
-	"testing"
-	"time"
-
-	"github.com/featurebasedb/featurebase/v3/statsd"
-	_ "github.com/featurebasedb/featurebase/v3/test"
-)
-
-func TestStatsClient_WithTags(t *testing.T) {
-	// Create a new client.
-	c, err := statsd.NewStatsClient("localhost:19444", "testnamespace")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer c.Close()
-
-	// Create a new client with additional tags.
-	c1 := c.WithTags("foo", "bar")
-	if tags := c1.Tags(); !reflect.DeepEqual(tags, []string{"bar", "foo"}) {
-		t.Fatalf("unexpected tags: %+v", tags)
-	}
-
-	// Create a new client from the clone with more tags.
-	c2 := c1.WithTags("bar", "baz")
-	if tags := c2.Tags(); !reflect.DeepEqual(tags, []string{"bar", "baz", "foo"}) {
-		t.Fatalf("unexpected tags: %+v", tags)
-	}
-}
-
-func TestStatsClient_Methods(t *testing.T) {
-	// Create a new client.
-	c, err := statsd.NewStatsClient("localhost:19444", "testnamespace")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer c.Close()
-
-	dur, _ := time.ParseDuration("123us")
-	c.CountWithCustomTags("ct", 1, 1.0, []string{"foo:bar"})
-	c.Count("cc", 1, 1.0)
-	c.Gauge("gg", 10, 1.0)
-	c.Histogram("hh", 1, 1.0)
-	c.Timing("tt", dur, 1.0)
-	c.Set("ss", "ss", 1.0)
-}
--- a/view.go
+++ b/view.go
@ -51,7 +51,6 @@ type view struct {
 	fragments map[uint64]*fragment

 	broadcaster broadcaster
-	stats       stats.StatsClient

 	knownShards       *roaring.Bitmap
 	knownShardsCopied uint32
@ -79,7 +78,6 @@ func newView(holder *Holder, path, index, field, name string, fieldOptions Field
 		fragments: make(map[uint64]*fragment),

 		broadcaster: NopBroadcaster,
-		stats:       stats.NopStatsClient,
 		knownShards: roaring.NewSliceBitmap(),

 		closing: make(chan struct{}),
@ -393,7 +391,6 @@ func (v *view) newFragment(shard uint64) *fragment {
 	frag := newFragment(v.holder, v.idx, v.fld, v, shard)
 	frag.CacheType = v.cacheType
 	frag.CacheSize = v.cacheSize
-	frag.stats = v.stats
 	if v.fieldType == FieldTypeMutex {
 		frag.mutexVector = newRowsVector(frag)
 	} else if v.fieldType == FieldTypeBool {
--- a/wireprotocol/wireprimitives.go
+++ b/wireprotocol/wireprimitives.go
@ -0,0 +1,563 @@
+package wireprotocol
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"time"
+
+	"io"
+
+	"github.com/featurebasedb/featurebase/v3/errors"
+	"github.com/featurebasedb/featurebase/v3/pql"
+	"github.com/featurebasedb/featurebase/v3/sql3/parser"
+	"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
+)
+
+const (
+	// server --> client
+	TOKEN_SCHEMA_INFO   int16 = 0xA1
+	TOKEN_ROW           int16 = 0xA2
+	TOKEN_DONE          int16 = 0xFD
+	TOKEN_INFO_MESSAGE  int16 = 0xFE
+	TOKEN_ERROR_MESSAGE int16 = 0xFF
+
+	// client --> server
+	TOKEN_SQL     int16 = 0x01
+	TOKEN_PLAN_OP int16 = 0x02
+)
+
+const (
+	TYPE_VOID      int8 = 0x00
+	TYPE_ID        int8 = 0x01
+	TYPE_BOOL      int8 = 0x02
+	TYPE_INT       int8 = 0x03
+	TYPE_DECIMAL   int8 = 0x04
+	TYPE_TIMESTAMP int8 = 0x05
+	TYPE_IDSET     int8 = 0x06
+	TYPE_STRING    int8 = 0x07
+	TYPE_STRINGSET int8 = 0x08
+)
+
+func ExpectToken(reader io.Reader, token int16) (int16, error) {
+	var tk int16
+	err := binary.Read(reader, binary.BigEndian, &tk)
+	if err != nil {
+		return 0, err
+	}
+	if tk != token {
+		return 0, errors.Errorf("expected token found %d", token)
+	}
+	return tk, nil
+}
+
+// TOKEN_COLUMN_INFO message
+// 					length (bytes)
+// token			2
+// column count		2
+//
+// (n) columns
+//
+// name length		1
+// name				(from prev)
+// data type		1
+//
+// (optional)
+// if type decimal
+// scale			1
+
+// note RelationName and AliasName members from
+// PlannerColumn are not sent over the wire
+func WriteSchema(schema types.Schema) ([]byte, error) {
+	buf := new(bytes.Buffer)
+	writer := bufio.NewWriter(buf)
+	// write token
+	writeToken(writer, TOKEN_SCHEMA_INFO)
+
+	// column count
+	writeInt16(writer, int16(len(schema)))
+
+	// for each column
+	for _, s := range schema {
+		// name len byte
+		writeInt8(writer, int8(len(s.ColumnName)))
+		// name
+		writer.WriteString(s.ColumnName)
+		// type byte
+		switch ty := s.Type.(type) {
+		case *parser.DataTypeID:
+			writeInt8(writer, TYPE_ID)
+
+		case *parser.DataTypeBool:
+			writeInt8(writer, TYPE_BOOL)
+
+		case *parser.DataTypeInt:
+			writeInt8(writer, TYPE_INT)
+
+		case *parser.DataTypeDecimal:
+			writeInt8(writer, TYPE_DECIMAL)
+			writeInt8(writer, int8(ty.Scale))
+
+		case *parser.DataTypeTimestamp:
+			writeInt8(writer, TYPE_TIMESTAMP)
+
+		case *parser.DataTypeIDSet:
+			writeInt8(writer, TYPE_IDSET)
+
+		case *parser.DataTypeString:
+			writeInt8(writer, TYPE_STRING)
+
+		case *parser.DataTypeStringSet:
+			writeInt8(writer, TYPE_STRINGSET)
+
+		default:
+			return []byte{}, errors.Errorf("unexpected type '%T'", s.Type)
+		}
+	}
+	writer.Flush()
+	return buf.Bytes(), nil
+}
+
+func ReadSchema(reader io.Reader) (types.Schema, error) {
+	var columnCount int16
+	err := binary.Read(reader, binary.BigEndian, &columnCount)
+	if err != nil {
+		return nil, err
+	}
+
+	var schema types.Schema
+	for i := 0; i < int(columnCount); i++ {
+
+		var nameLen int8
+		err = binary.Read(reader, binary.BigEndian, &nameLen)
+		if err != nil {
+			return nil, err
+		}
+		bname := make([]byte, nameLen)
+		err = binary.Read(reader, binary.BigEndian, &bname)
+		if err != nil {
+			return nil, err
+		}
+		colName := string(bname)
+
+		var typ int8
+		err = binary.Read(reader, binary.BigEndian, &typ)
+		if err != nil {
+			return nil, err
+		}
+		var dataType parser.ExprDataType
+
+		switch typ {
+		case TYPE_ID:
+			dataType = parser.NewDataTypeID()
+
+		case TYPE_BOOL:
+			dataType = parser.NewDataTypeBool()
+
+		case TYPE_INT:
+			dataType = parser.NewDataTypeInt()
+
+		case TYPE_DECIMAL:
+			var scale int8
+			err = binary.Read(reader, binary.BigEndian, &scale)
+			if err != nil {
+				return nil, err
+			}
+			dataType = parser.NewDataTypeDecimal(int64(scale))
+
+		case TYPE_TIMESTAMP:
+			dataType = parser.NewDataTypeTimestamp()
+
+		case TYPE_IDSET:
+			dataType = parser.NewDataTypeIDSet()
+
+		case TYPE_STRING:
+			dataType = parser.NewDataTypeString()
+
+		case TYPE_STRINGSET:
+			dataType = parser.NewDataTypeStringSet()
+		}
+
+		schema = append(schema, &types.PlannerColumn{
+			ColumnName: colName,
+			Type:       dataType,
+		})
+	}
+	return schema, nil
+}
+
+// TOKEN_ROW message
+// 					length (bytes)
+// token			2
+//
+// (n) columns
+// if column length is 0 --> null
+//
+// - for ID, INT
+//
+// column length	1
+// value			8
+//
+// - for DECIMAL
+//
+// column length	1
+// value			8
+//
+// - for BOOL
+//
+// column length	1
+// value			1
+//
+// - for TIMESTAMP
+//
+// column length	1
+// value			8
+//
+// - for IDSET
+//
+// set length		2
+// (n) values
+// 		value		8
+//
+// - for STRING
+//
+// column length	2
+// value			(from prev)
+//
+// - for STRINGSET
+//
+// set length		2
+// (n) values
+// 		value len	2
+// 		value		(from prev)
+//
+
+func WriteRow(row types.Row, schema types.Schema) ([]byte, error) {
+	buf := new(bytes.Buffer)
+	writer := bufio.NewWriter(buf)
+	// write token
+	writeToken(writer, TOKEN_ROW)
+
+	// for each column
+	for i, s := range schema {
+		val := row[i]
+		switch s.Type.(type) {
+		case *parser.DataTypeID, *parser.DataTypeInt:
+			if val == nil {
+				writeInt8(writer, 0)
+			} else {
+				writeInt8(writer, 8)
+				v, ok := row[i].(int64)
+				if !ok {
+					return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
+				}
+				writeInt64(writer, v)
+			}
+
+		case *parser.DataTypeDecimal:
+			if val == nil {
+				writeInt8(writer, 0)
+			} else {
+				writeInt8(writer, 8)
+				v, ok := row[i].(pql.Decimal)
+				if !ok {
+					return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
+				}
+				writeInt64(writer, v.ToInt64(v.Scale))
+			}
+
+		case *parser.DataTypeBool:
+			if val == nil {
+				writeInt8(writer, 0)
+			} else {
+				writeInt8(writer, 1)
+				v, ok := row[i].(bool)
+				if !ok {
+					return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
+				}
+				if v {
+					writeInt8(writer, 1)
+				} else {
+					writeInt8(writer, 0)
+				}
+			}
+
+		case *parser.DataTypeTimestamp:
+			if val == nil {
+				writeInt8(writer, 0)
+			} else {
+				writeInt8(writer, 8)
+				v, ok := row[i].(time.Time)
+				if !ok {
+					return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
+				}
+				writeInt64(writer, v.UnixNano())
+			}
+
+		case *parser.DataTypeIDSet:
+			if val == nil {
+				writeInt16(writer, 0)
+			} else {
+				v, ok := row[i].([]int64)
+				if !ok {
+					return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
+				}
+				writeInt16(writer, int16(len(v)))
+				for _, s := range v {
+					writeInt64(writer, s)
+				}
+			}
+
+		case *parser.DataTypeString:
+			if val == nil {
+				writeInt16(writer, 0)
+			} else {
+				v, ok := row[i].(string)
+				if !ok {
+					return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
+				}
+				writeInt16(writer, int16(len(v)))
+				writer.WriteString(v)
+			}
+
+		case *parser.DataTypeStringSet:
+			if val == nil {
+				writeInt16(writer, 0)
+			} else {
+				v, ok := row[i].([]string)
+				if !ok {
+					return []byte{}, errors.Errorf("unexpected type '%T'", row[i])
+				}
+				writeInt16(writer, int16(len(v)))
+				for _, s := range v {
+					writeInt16(writer, int16(len(s)))
+					writer.WriteString(s)
+				}
+			}
+
+		default:
+			return []byte{}, errors.Errorf("unexpected type '%T'", s.Type)
+		}
+	}
+	writer.Flush()
+	return buf.Bytes(), nil
+}
+
+func ReadRow(reader io.Reader, schema types.Schema) (types.Row, error) {
+
+	row := make(types.Row, len(schema))
+
+	for idx, s := range schema {
+		switch t := s.Type.(type) {
+		case *parser.DataTypeID, *parser.DataTypeInt:
+			var len int8
+			err := binary.Read(reader, binary.BigEndian, &len)
+			if err != nil {
+				return nil, err
+			}
+			if len == 0 {
+				row[idx] = nil
+			} else {
+				var value int64
+				err := binary.Read(reader, binary.BigEndian, &value)
+				if err != nil {
+					return nil, err
+				}
+				row[idx] = value
+			}
+
+		case *parser.DataTypeDecimal:
+			var len int8
+			err := binary.Read(reader, binary.BigEndian, &len)
+			if err != nil {
+				return nil, err
+			}
+			if len == 0 {
+				row[idx] = nil
+			} else {
+				var value int64
+				err := binary.Read(reader, binary.BigEndian, &value)
+				if err != nil {
+					return nil, err
+				}
+				row[idx] = pql.NewDecimal(value, t.Scale)
+			}
+
+		case *parser.DataTypeBool:
+			var len int8
+			err := binary.Read(reader, binary.BigEndian, &len)
+			if err != nil {
+				return nil, err
+			}
+			if len == 0 {
+				row[idx] = nil
+			} else {
+				var value int8
+				err := binary.Read(reader, binary.BigEndian, &value)
+				if err != nil {
+					return nil, err
+				}
+				row[idx] = value == 1
+			}
+
+		case *parser.DataTypeTimestamp:
+			var len int8
+			err := binary.Read(reader, binary.BigEndian, &len)
+			if err != nil {
+				return nil, err
+			}
+			if len == 0 {
+				row[idx] = nil
+			} else {
+				var value int64
+				err := binary.Read(reader, binary.BigEndian, &value)
+				if err != nil {
+					return nil, err
+				}
+				row[idx] = time.Unix(0, value)
+			}
+
+		case *parser.DataTypeIDSet:
+			var len int16
+			err := binary.Read(reader, binary.BigEndian, &len)
+			if err != nil {
+				return nil, err
+			}
+			if len == 0 {
+				row[idx] = nil
+			} else {
+				set := make([]int64, len)
+				for j, _ := range set {
+					var value int64
+					err := binary.Read(reader, binary.BigEndian, &value)
+					if err != nil {
+						return nil, err
+					}
+					set[j] = value
+				}
+				row[idx] = set
+			}
+
+		case *parser.DataTypeString:
+			var len int16
+			err := binary.Read(reader, binary.BigEndian, &len)
+			if err != nil {
+				return nil, err
+			}
+			if len == 0 {
+				row[idx] = nil
+			} else {
+				bvalue := make([]byte, len)
+				err = binary.Read(reader, binary.BigEndian, &bvalue)
+				if err != nil {
+					return nil, err
+				}
+				row[idx] = string(bvalue)
+			}
+
+		case *parser.DataTypeStringSet:
+			var len int16
+			err := binary.Read(reader, binary.BigEndian, &len)
+			if err != nil {
+				return nil, err
+			}
+			if len == 0 {
+				row[idx] = nil
+			} else {
+				set := make([]string, len)
+				for j, _ := range set {
+					var vlen int16
+					err = binary.Read(reader, binary.BigEndian, &vlen)
+					if err != nil {
+						return nil, err
+					}
+					bvalue := make([]byte, vlen)
+					err = binary.Read(reader, binary.BigEndian, &bvalue)
+					if err != nil {
+						return nil, err
+					}
+					set[j] = string(bvalue)
+				}
+				row[idx] = set
+			}
+
+		default:
+			return nil, errors.Errorf("unexpected type '%T'", s.Type)
+		}
+	}
+
+	return row, nil
+}
+
+// TOKEN_DONE message
+// 					length (bytes)
+// token			2
+
+func WriteDone() []byte {
+	buf := new(bytes.Buffer)
+	writer := bufio.NewWriter(buf)
+	// write token
+	writeToken(writer, TOKEN_DONE)
+	writer.Flush()
+	return buf.Bytes()
+}
+
+// TOKEN_ERROR_MESSAGE message
+// 					length (bytes)
+// token			2
+//
+// message len		4
+// message			(from prev)
+
+func WriteError(err error) []byte {
+	buf := new(bytes.Buffer)
+	writer := bufio.NewWriter(buf)
+	// write token
+	writeToken(writer, TOKEN_ERROR_MESSAGE)
+	// write error len
+	t := err.Error()
+	b := make([]byte, 4)
+	binary.BigEndian.PutUint32(b, uint32(len(t)))
+	writer.Write(b)
+	// write error
+	writer.WriteString(t)
+	writer.Flush()
+
+	return buf.Bytes()
+}
+
+func ReadError(reader io.Reader) (string, error) {
+	var len int16
+	err := binary.Read(reader, binary.BigEndian, &len)
+	if err != nil {
+		return "", err
+	}
+	bvalue := make([]byte, len)
+	err = binary.Read(reader, binary.BigEndian, &bvalue)
+	if err != nil {
+		return "", err
+	}
+	return string(bvalue), nil
+}
+
+func writeToken(w io.Writer, token int16) {
+	writeInt16(w, token)
+}
+
+func writeInt8(w io.Writer, i int8) {
+	b := make([]byte, 1)
+	b[0] = byte(i)
+	w.Write(b)
+}
+
+func writeInt16(w io.Writer, i int16) {
+	b := make([]byte, 2)
+	binary.BigEndian.PutUint16(b, uint16(i))
+	w.Write(b)
+}
+
+func writeInt64(w io.Writer, i int64) {
+	b := make([]byte, 8)
+	binary.BigEndian.PutUint64(b, uint64(i))
+	w.Write(b)
+}
--- a/wireprotocol/wireprimitives_test.go
+++ b/wireprotocol/wireprimitives_test.go
@ -0,0 +1,144 @@
+package wireprotocol_test
+
+import (
+	"bytes"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/featurebasedb/featurebase/v3/pql"
+	"github.com/featurebasedb/featurebase/v3/sql3/parser"
+	"github.com/featurebasedb/featurebase/v3/sql3/planner/types"
+	"github.com/featurebasedb/featurebase/v3/wireprotocol"
+)
+
+func TestWireProtocol_Schema(t *testing.T) {
+
+	s := types.Schema{
+		&types.PlannerColumn{
+			ColumnName: "col1",
+			Type:       parser.NewDataTypeID(),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col2",
+			Type:       parser.NewDataTypeInt(),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col3",
+			Type:       parser.NewDataTypeDecimal(4),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col4",
+			Type:       parser.NewDataTypeString(),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col5",
+			Type:       parser.NewDataTypeStringSet(),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col6",
+			Type:       parser.NewDataTypeIDSet(),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col7",
+			Type:       parser.NewDataTypeBool(),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col8",
+			Type:       parser.NewDataTypeTimestamp(),
+		},
+	}
+
+	b, err := wireprotocol.WriteSchema(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rdr := bytes.NewReader(b)
+	_, err = wireprotocol.ExpectToken(rdr, wireprotocol.TOKEN_SCHEMA_INFO)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sr, err := wireprotocol.ReadSchema(rdr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if diff := cmp.Diff(s, sr); diff != "" {
+		t.Fatal(diff)
+	}
+}
+
+func TestWireProtocol_Row(t *testing.T) {
+
+	s := types.Schema{
+		&types.PlannerColumn{
+			ColumnName: "col1",
+			Type:       parser.NewDataTypeID(),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col2",
+			Type:       parser.NewDataTypeInt(),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col3",
+			Type:       parser.NewDataTypeDecimal(4),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col4",
+			Type:       parser.NewDataTypeString(),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col5",
+			Type:       parser.NewDataTypeStringSet(),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col6",
+			Type:       parser.NewDataTypeIDSet(),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col7",
+			Type:       parser.NewDataTypeBool(),
+		},
+		&types.PlannerColumn{
+			ColumnName: "col8",
+			Type:       parser.NewDataTypeTimestamp(),
+		},
+	}
+
+	r := types.Row{
+		int64(1),
+		int64(2),
+		pql.NewDecimal(123400, 4),
+		string("foo"),
+		[]string{"bar", "baz"},
+		[]int64{10, 20},
+		bool(false),
+		time.Now().UTC(),
+	}
+
+	b, err := wireprotocol.WriteRow(r, s)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rdr := bytes.NewReader(b)
+	_, err = wireprotocol.ExpectToken(rdr, wireprotocol.TOKEN_ROW)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rr, err := wireprotocol.ReadRow(rdr, s)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	opt := cmp.Comparer(func(x, y pql.Decimal) bool {
+		return x.EqualTo(y)
+	})
+
+	if diff := cmp.Diff(r, rr, opt); diff != "" {
+		t.Fatal(diff)
+	}
+}